diff --git a/FWCore/Integration/bin/interprocess.cc b/FWCore/Integration/bin/interprocess.cc index 1417c694bdc0f..09c469d608073 100644 --- a/FWCore/Integration/bin/interprocess.cc +++ b/FWCore/Integration/bin/interprocess.cc @@ -115,7 +115,7 @@ int main(int argc, char* argv[]) { //This class is holding the lock WorkerChannel communicationChannel(memoryName, uniqueID); - WriteBuffer sm_buffer{memoryName, communicationChannel.fromWorkerBufferIndex()}; + WriteBuffer sm_buffer{memoryName, communicationChannel.fromWorkerBufferInfo()}; int counter = 0; //The lock must be released if there is a catastrophic signal diff --git a/FWCore/Integration/bin/interprocess_random.cc b/FWCore/Integration/bin/interprocess_random.cc index 9f15ef65ff9bd..82a3f2e5e4b04 100644 --- a/FWCore/Integration/bin/interprocess_random.cc +++ b/FWCore/Integration/bin/interprocess_random.cc @@ -104,8 +104,8 @@ int main(int argc, char* argv[]) { //This class is holding the lock WorkerChannel communicationChannel(memoryName, uniqueID); - WriteBuffer sm_buffer{memoryName, communicationChannel.fromWorkerBufferIndex()}; - ReadBuffer sm_readbuffer{std::string("Rand") + memoryName, communicationChannel.toWorkerBufferIndex()}; + WriteBuffer sm_buffer{memoryName, communicationChannel.fromWorkerBufferInfo()}; + ReadBuffer sm_readbuffer{std::string("Rand") + memoryName, communicationChannel.toWorkerBufferInfo()}; int counter = 0; //The lock must be released if there is a catastrophic signal diff --git a/FWCore/Integration/test/TestInterProcessProd.cc b/FWCore/Integration/test/TestInterProcessProd.cc index 77bfc96563536..26d256aabbcd8 100644 --- a/FWCore/Integration/test/TestInterProcessProd.cc +++ b/FWCore/Integration/test/TestInterProcessProd.cc @@ -20,7 +20,7 @@ namespace testinter { StreamCache(const std::string& iConfig, int id) : id_{id}, channel_("testProd", id_), - readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferIndex()}, + readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferInfo()}, deserializer_{readBuffer_}, br_deserializer_{readBuffer_}, er_deserializer_{readBuffer_}, diff --git a/FWCore/Integration/test/TestInterProcessRandomProd.cc b/FWCore/Integration/test/TestInterProcessRandomProd.cc index bb0088006d302..95c42023b8484 100644 --- a/FWCore/Integration/test/TestInterProcessRandomProd.cc +++ b/FWCore/Integration/test/TestInterProcessRandomProd.cc @@ -32,8 +32,8 @@ namespace testinter { StreamCache(const std::string& iConfig, int id) : id_{id}, channel_("testProd", id_), - readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferIndex()}, - writeBuffer_{std::string("Rand") + channel_.sharedMemoryName(), channel_.toWorkerBufferIndex()}, + readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferInfo()}, + writeBuffer_{std::string("Rand") + channel_.sharedMemoryName(), channel_.toWorkerBufferInfo()}, deserializer_{readBuffer_}, bl_deserializer_{readBuffer_}, randSerializer_{writeBuffer_} { diff --git a/FWCore/SharedMemory/interface/BufferInfo.h b/FWCore/SharedMemory/interface/BufferInfo.h new file mode 100644 index 0000000000000..745cab37d14de --- /dev/null +++ b/FWCore/SharedMemory/interface/BufferInfo.h @@ -0,0 +1,33 @@ +#ifndef FWCore_SharedMemory_BufferInfo_h +#define FWCore_SharedMemory_BufferInfo_h +// -*- C++ -*- +// +// Package: FWCore/SharedMemory +// Class : BufferInfo +// +/**\class BufferInfo BufferInfo.h " FWCore/SharedMemory/interface/BufferInfo.h" + + Description: Information needed to manage the buffer + + Usage: + This is an internal detail of the system. +*/ +// +// Original Author: Chris Jones +// Created: 21/01/2020 +// + +// system include files + +// user include files + +// forward declarations + +namespace edm::shared_memory { + struct BufferInfo { + int identifier_; + char index_; + }; +} // namespace edm::shared_memory + +#endif diff --git a/FWCore/SharedMemory/interface/ControllerChannel.h b/FWCore/SharedMemory/interface/ControllerChannel.h index 544a9c1745362..8c189ae1e814d 100644 --- a/FWCore/SharedMemory/interface/ControllerChannel.h +++ b/FWCore/SharedMemory/interface/ControllerChannel.h @@ -29,6 +29,7 @@ // user include files #include "FWCore/Utilities/interface/Transition.h" #include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/SharedMemory/interface/BufferInfo.h" // forward declarations @@ -48,7 +49,7 @@ namespace edm::shared_memory { // ---------- member functions --------------------------- /** setupWorker must be called only once and done before any calls to doTransition. The functor iF should setup values associated - with shared memory use, such as manipulating the value from toWorkerBufferIndex(). The call to setupWorker proper synchronizes + with shared memory use, such as manipulating the value from toWorkerBufferInfo(). The call to setupWorker proper synchronizes the Controller and Worker processes. */ template @@ -83,9 +84,9 @@ namespace edm::shared_memory { } ///This can be used with WriteBuffer to keep Controller and Worker in sync - char* toWorkerBufferIndex() { return toWorkerBufferIndex_; } + BufferInfo* toWorkerBufferInfo() { return toWorkerBufferInfo_; } ///This can be used with ReadBuffer to keep Controller and Worker in sync - char* fromWorkerBufferIndex() { return fromWorkerBufferIndex_; } + BufferInfo* fromWorkerBufferInfo() { return fromWorkerBufferInfo_; } void stopWorker() { //std::cout <<"stopWorker"<ReadBuffer(bufferFile_, &value); @@ -58,6 +59,7 @@ namespace edm::shared_memory { READBUFFER& buffer_; TClass* const class_; TBufferFile bufferFile_; + int previousBufferIdentifier_ = 0; }; } // namespace edm::shared_memory diff --git a/FWCore/SharedMemory/interface/ReadBuffer.h b/FWCore/SharedMemory/interface/ReadBuffer.h index b3fd8bc67dbbe..a58b326b241ba 100644 --- a/FWCore/SharedMemory/interface/ReadBuffer.h +++ b/FWCore/SharedMemory/interface/ReadBuffer.h @@ -26,6 +26,7 @@ This works in conjunction with WriteBuffer. // user include files #include "FWCore/SharedMemory/interface/buffer_names.h" +#include "FWCore/SharedMemory/interface/BufferInfo.h" // forward declarations @@ -33,11 +34,11 @@ namespace edm::shared_memory { class ReadBuffer { public: /** iUniqueName : must be unique for all processes running on a system. - iBufferIndex : is a pointer to a shared_memory address where the same address needs to be shared by ReadBuffer and WriteBuffer. + iBufferInfo : is a pointer to a shared_memory address where the same address needs to be shared by ReadBuffer and WriteBuffer. */ - ReadBuffer(std::string const& iUniqueName, char* iBufferIndex) - : buffer_{nullptr, 0}, bufferIndex_{iBufferIndex}, bufferOldIndex_{3} { - *bufferIndex_ = 0; + ReadBuffer(std::string const& iUniqueName, BufferInfo* iBufferInfo) + : buffer_{nullptr, 0}, bufferInfo_{iBufferInfo}, bufferOldIndex_{3} { + *bufferInfo_ = {0, 0}; bufferNames_[0] = iUniqueName + buffer_names::kBuffer0; bufferNames_[1] = iUniqueName + buffer_names::kBuffer1; } @@ -47,23 +48,24 @@ namespace edm::shared_memory { const ReadBuffer& operator=(ReadBuffer&&) = delete; // ---------- const member functions --------------------- - bool mustGetBufferAgain() const { return *bufferIndex_ != bufferOldIndex_; } + int bufferIdentifier() const { return bufferInfo_->identifier_; } // ---------- member functions --------------------------- std::pair buffer() { if (mustGetBufferAgain()) { using namespace boost::interprocess; - sm_ = std::make_unique(open_only, bufferNames_[*bufferIndex_].c_str()); + sm_ = std::make_unique(open_only, bufferNames_[bufferInfo_->index_].c_str()); buffer_ = sm_->find(buffer_names::kBuffer); - bufferOldIndex_ = *bufferIndex_; + bufferOldIndex_ = bufferInfo_->index_; } return buffer_; } private: + bool mustGetBufferAgain() const { return bufferInfo_->index_ != bufferOldIndex_; } // ---------- member data -------------------------------- std::pair buffer_; - char* bufferIndex_; + BufferInfo* bufferInfo_; char bufferOldIndex_; std::array bufferNames_; std::unique_ptr sm_; diff --git a/FWCore/SharedMemory/interface/WorkerChannel.h b/FWCore/SharedMemory/interface/WorkerChannel.h index 22e3cae178df9..9e7aa42c257fa 100644 --- a/FWCore/SharedMemory/interface/WorkerChannel.h +++ b/FWCore/SharedMemory/interface/WorkerChannel.h @@ -27,6 +27,7 @@ // user include files #include "FWCore/Utilities/interface/Transition.h" +#include "FWCore/SharedMemory/interface/BufferInfo.h" // forward declarations @@ -47,9 +48,9 @@ namespace edm::shared_memory { boost::interprocess::scoped_lock* accessLock() { return &lock_; } ///This can be used with ReadBuffer to keep Controller and Worker in sync - char* toWorkerBufferIndex() { return toWorkerBufferIndex_; } + BufferInfo* toWorkerBufferInfo() { return toWorkerBufferInfo_; } ///This can be used with WriteBuffer to keep Controller and Worker in sync - char* fromWorkerBufferIndex() { return fromWorkerBufferIndex_; } + BufferInfo* fromWorkerBufferInfo() { return fromWorkerBufferInfo_; } ///Matches the ControllerChannel::setupWorker call void workerSetupDone() { @@ -94,8 +95,8 @@ namespace edm::shared_memory { bool* stop_; edm::Transition* transitionType_; unsigned long long* transitionID_; - char* toWorkerBufferIndex_; - char* fromWorkerBufferIndex_; + BufferInfo* toWorkerBufferInfo_; + BufferInfo* fromWorkerBufferInfo_; boost::interprocess::named_condition cndToController_; bool* keepEvent_; boost::interprocess::scoped_lock lock_; diff --git a/FWCore/SharedMemory/interface/WriteBuffer.h b/FWCore/SharedMemory/interface/WriteBuffer.h index 028040654a3e6..1080d678df262 100644 --- a/FWCore/SharedMemory/interface/WriteBuffer.h +++ b/FWCore/SharedMemory/interface/WriteBuffer.h @@ -29,6 +29,7 @@ // user include files #include "FWCore/SharedMemory/interface/buffer_names.h" +#include "FWCore/SharedMemory/interface/BufferInfo.h" // forward declarations @@ -36,14 +37,14 @@ namespace edm::shared_memory { class WriteBuffer { public: /** iUniqueName : must be unique for all processes running on a system. - iBufferIndex : is a pointer to a shared_memory address where the same address needs to be shared by ReadBuffer and WriteBuffer. + iBufferInfo : is a pointer to a shared_memory address where the same address needs to be shared by ReadBuffer and WriteBuffer. */ - WriteBuffer(std::string const& iUniqueName, char* iBufferIndex) - : bufferSize_{0}, buffer_{nullptr}, bufferIndex_{iBufferIndex} { + WriteBuffer(std::string const& iUniqueName, BufferInfo* iBufferInfo) + : bufferSize_{0}, buffer_{nullptr}, bufferInfo_{iBufferInfo} { bufferNames_[0] = iUniqueName + buffer_names::kBuffer0; bufferNames_[1] = iUniqueName + buffer_names::kBuffer1; - assert(bufferIndex_); + assert(bufferInfo_); } WriteBuffer(const WriteBuffer&) = delete; const WriteBuffer& operator=(const WriteBuffer&) = delete; @@ -66,7 +67,7 @@ namespace edm::shared_memory { // ---------- member data -------------------------------- std::size_t bufferSize_; char* buffer_; - char* bufferIndex_; + BufferInfo* bufferInfo_; std::array bufferNames_; std::unique_ptr sm_; }; diff --git a/FWCore/SharedMemory/interface/channel_names.h b/FWCore/SharedMemory/interface/channel_names.h index 7583745a2a69d..8d369e36c2732 100644 --- a/FWCore/SharedMemory/interface/channel_names.h +++ b/FWCore/SharedMemory/interface/channel_names.h @@ -26,8 +26,8 @@ namespace edm::shared_memory { namespace channel_names { - constexpr char const* const kToWorkerBufferIndex = "bufferIndexToWorker"; - constexpr char const* const kFromWorkerBufferIndex = "bufferIndexFromWorker"; + constexpr char const* const kToWorkerBufferInfo = "bufferInfoToWorker"; + constexpr char const* const kFromWorkerBufferInfo = "bufferInfoFromWorker"; constexpr char const* const kMutex = "mtx"; constexpr char const* const kConditionFromMain = "cndFromMain"; constexpr char const* const kConditionToMain = "cndToMain"; diff --git a/FWCore/SharedMemory/src/ControllerChannel.cc b/FWCore/SharedMemory/src/ControllerChannel.cc index 89e02361f209a..184796737e20b 100644 --- a/FWCore/SharedMemory/src/ControllerChannel.cc +++ b/FWCore/SharedMemory/src/ControllerChannel.cc @@ -35,8 +35,8 @@ ControllerChannel::ControllerChannel(std::string const& iName, int id) : id_{id}, smName_{uniqueName(iName)}, managed_sm_{open_or_create, smName_.c_str(), 1024}, - toWorkerBufferIndex_{bufferIndex(channel_names::kToWorkerBufferIndex, managed_sm_)}, - fromWorkerBufferIndex_{bufferIndex(channel_names::kFromWorkerBufferIndex, managed_sm_)}, + toWorkerBufferInfo_{bufferInfo(channel_names::kToWorkerBufferInfo, managed_sm_)}, + fromWorkerBufferInfo_{bufferInfo(channel_names::kFromWorkerBufferInfo, managed_sm_)}, mutex_{open_or_create, uniqueName(channel_names::kMutex).c_str()}, cndFromMain_{open_or_create, uniqueName(channel_names::kConditionFromMain).c_str()}, cndToMain_{open_or_create, uniqueName(channel_names::kConditionToMain).c_str()} { @@ -61,8 +61,8 @@ ControllerChannel::~ControllerChannel() { managed_sm_.destroy(channel_names::kStop); managed_sm_.destroy(channel_names::kTransitionType); managed_sm_.destroy(channel_names::kTransitionID); - managed_sm_.destroy(channel_names::kToWorkerBufferIndex); - managed_sm_.destroy(channel_names::kFromWorkerBufferIndex); + managed_sm_.destroy(channel_names::kToWorkerBufferInfo); + managed_sm_.destroy(channel_names::kFromWorkerBufferInfo); named_mutex::remove(uniqueName(channel_names::kMutex).c_str()); named_condition::remove(uniqueName(channel_names::kConditionFromMain).c_str()); @@ -103,8 +103,8 @@ bool ControllerChannel::wait(scoped_lock& lock, edm::Transition iTr // // static member functions // -char* ControllerChannel::bufferIndex(const char* iWhich, managed_shared_memory& mem) { - mem.destroy(iWhich); - char* v = mem.construct(iWhich)(); +BufferInfo* ControllerChannel::bufferInfo(const char* iWhich, managed_shared_memory& mem) { + mem.destroy(iWhich); + BufferInfo* v = mem.construct(iWhich)(); return v; } diff --git a/FWCore/SharedMemory/src/WorkerChannel.cc b/FWCore/SharedMemory/src/WorkerChannel.cc index 5fe1f5c7e0dc4..2659646b88e90 100644 --- a/FWCore/SharedMemory/src/WorkerChannel.cc +++ b/FWCore/SharedMemory/src/WorkerChannel.cc @@ -44,16 +44,16 @@ WorkerChannel::WorkerChannel(std::string const& iName, const std::string& iUniqu stop_{managed_shm_.find(channel_names::kStop).first}, transitionType_{managed_shm_.find(channel_names::kTransitionType).first}, transitionID_{managed_shm_.find(channel_names::kTransitionID).first}, - toWorkerBufferIndex_{managed_shm_.find(channel_names::kToWorkerBufferIndex).first}, - fromWorkerBufferIndex_{managed_shm_.find(channel_names::kFromWorkerBufferIndex).first}, + toWorkerBufferInfo_{managed_shm_.find(channel_names::kToWorkerBufferInfo).first}, + fromWorkerBufferInfo_{managed_shm_.find(channel_names::kFromWorkerBufferInfo).first}, cndToController_{open_or_create, unique_name(channel_names::kConditionToMain, iUniqueID).c_str()}, keepEvent_{managed_shm_.find(channel_names::kKeepEvent).first}, lock_{mutex_} { assert(stop_); assert(transitionType_); assert(transitionID_); - assert(toWorkerBufferIndex_); - assert(fromWorkerBufferIndex_); + assert(toWorkerBufferInfo_); + assert(fromWorkerBufferInfo_); } // diff --git a/FWCore/SharedMemory/src/WriteBuffer.cc b/FWCore/SharedMemory/src/WriteBuffer.cc index c5bbb9eea7787..3713c0e8cf3cd 100644 --- a/FWCore/SharedMemory/src/WriteBuffer.cc +++ b/FWCore/SharedMemory/src/WriteBuffer.cc @@ -32,24 +32,25 @@ WriteBuffer::~WriteBuffer() { if (sm_) { sm_->destroy(buffer_names::kBuffer); sm_.reset(); - boost::interprocess::shared_memory_object::remove(bufferNames_[*bufferIndex_].c_str()); + boost::interprocess::shared_memory_object::remove(bufferNames_[bufferInfo_->index_].c_str()); } } // // member functions // void WriteBuffer::growBuffer(std::size_t iLength) { - int newBuffer = (*bufferIndex_ + 1) % 2; + int newBuffer = (bufferInfo_->index_ + 1) % 2; if (sm_) { sm_->destroy(buffer_names::kBuffer); sm_.reset(); - boost::interprocess::shared_memory_object::remove(bufferNames_[*bufferIndex_].c_str()); + boost::interprocess::shared_memory_object::remove(bufferNames_[bufferInfo_->index_].c_str()); } sm_ = std::make_unique( boost::interprocess::open_or_create, bufferNames_[newBuffer].c_str(), iLength + 1024); assert(sm_.get()); bufferSize_ = iLength; - *bufferIndex_ = newBuffer; + bufferInfo_->index_ = newBuffer; + bufferInfo_->identifier_ = bufferInfo_->identifier_ + 1; buffer_ = sm_->construct(buffer_names::kBuffer)[iLength](0); assert(buffer_); } diff --git a/FWCore/SharedMemory/test/test_catch2_buffer.cc b/FWCore/SharedMemory/test/test_catch2_buffer.cc index 4d9bea8a43e47..5b99f8887f63a 100644 --- a/FWCore/SharedMemory/test/test_catch2_buffer.cc +++ b/FWCore/SharedMemory/test/test_catch2_buffer.cc @@ -9,20 +9,20 @@ using namespace edm::shared_memory; TEST_CASE("test Read/WriteBuffers", "[Buffers]") { - char bufferIndex = 0; + BufferInfo bufferInfo = {0, 0}; std::string const uniqueName = "BufferTest" + std::to_string(getpid()); - WriteBuffer writeBuffer(uniqueName, &bufferIndex); + WriteBuffer writeBuffer(uniqueName, &bufferInfo); - ReadBuffer readBuffer(uniqueName, &bufferIndex); + ReadBuffer readBuffer(uniqueName, &bufferInfo); SECTION("First") { std::array dummy = {{'t', 'e', 's', 't'}}; writeBuffer.copyToBuffer(dummy.data(), dummy.size()); - REQUIRE(readBuffer.mustGetBufferAgain()); + REQUIRE(readBuffer.bufferIdentifier() == 1); { auto b = readBuffer.buffer(); REQUIRE(b.second == dummy.size()); @@ -34,7 +34,7 @@ TEST_CASE("test Read/WriteBuffers", "[Buffers]") { writeBuffer.copyToBuffer(dummy.data(), dummy.size() - 1); - REQUIRE(not readBuffer.mustGetBufferAgain()); + REQUIRE(readBuffer.bufferIdentifier() == 1); { auto b = readBuffer.buffer(); //the second argument is the buffer capacity, not the last length sent @@ -46,7 +46,7 @@ TEST_CASE("test Read/WriteBuffers", "[Buffers]") { std::array dummy = {{'l', 'a', 'r', 'g', 'e', 'r'}}; writeBuffer.copyToBuffer(dummy.data(), dummy.size()); - REQUIRE(readBuffer.mustGetBufferAgain()); + REQUIRE(readBuffer.bufferIdentifier() == 2); { auto b = readBuffer.buffer(); REQUIRE(b.second == dummy.size()); @@ -58,7 +58,7 @@ TEST_CASE("test Read/WriteBuffers", "[Buffers]") { std::array dummy = {{'l', 'a', 'r', 'g', 'e', 's', 't'}}; writeBuffer.copyToBuffer(dummy.data(), dummy.size()); - REQUIRE(readBuffer.mustGetBufferAgain()); + REQUIRE(readBuffer.bufferIdentifier() == 3); { auto b = readBuffer.buffer(); REQUIRE(b.second == dummy.size()); diff --git a/FWCore/SharedMemory/test/test_catch2_serialize.cc b/FWCore/SharedMemory/test/test_catch2_serialize.cc index b6ef55f4cb825..1e7cffb580b7a 100644 --- a/FWCore/SharedMemory/test/test_catch2_serialize.cc +++ b/FWCore/SharedMemory/test/test_catch2_serialize.cc @@ -12,15 +12,13 @@ namespace { struct ReadWriteTestBuffer { std::pair buffer() { return std::pair(&buffer_.front(), size()); } - bool mustGetBufferAgain() { return resized_; } + int bufferIdentifier() { return bufferIdentifier_; } void copyToBuffer(char* iStart, std::size_t iLength) { buffer_.clear(); if (iLength > buffer_.capacity()) { buffer_.reserve(iLength); - resized_ = true; - } else { - resized_ = false; + ++bufferIdentifier_; } std::copy(iStart, iStart + iLength, std::back_insert_iterator(buffer_)); } @@ -28,7 +26,7 @@ namespace { std::size_t size() const { return buffer_.size(); } std::vector buffer_; - bool resized_ = true; + int bufferIdentifier_ = 1; }; bool compare(std::vector const& iLHS, std::vector const& iRHS) { @@ -56,7 +54,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { t.a = 42; serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == true); + REQUIRE(buffer.bufferIdentifier() == 2); auto newT = deserializer.deserialize(); @@ -64,7 +62,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { SECTION("Reuse buffer") { t.a = 12; serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == false); + REQUIRE(buffer.bufferIdentifier() == 2); auto newT = deserializer.deserialize(); @@ -87,7 +85,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { } serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == true); + REQUIRE(buffer.bufferIdentifier() == 2); auto newT = deserializer.deserialize(); @@ -97,7 +95,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { v.a += 1; } serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == false); + REQUIRE(buffer.bufferIdentifier() == 2); auto newT = deserializer.deserialize(); @@ -107,7 +105,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { SECTION("Grow") { t.emplace_back(); serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == true); + REQUIRE(buffer.bufferIdentifier() == 3); auto newT = deserializer.deserialize(); @@ -119,7 +117,7 @@ TEST_CASE("test De/ROOTSerializer", "[ROOTSerializer]") { t.pop_back(); serializer.serialize(t); - REQUIRE(buffer.mustGetBufferAgain() == false); + REQUIRE(buffer.bufferIdentifier() == 2); auto newT = deserializer.deserialize(); diff --git a/FWCore/SharedMemory/test/test_channels.cc b/FWCore/SharedMemory/test/test_channels.cc index d0e49ad698b84..82ec63221102d 100644 --- a/FWCore/SharedMemory/test/test_channels.cc +++ b/FWCore/SharedMemory/test/test_channels.cc @@ -38,12 +38,16 @@ namespace { }); } { - *channel.toWorkerBufferIndex() = 0; + *channel.toWorkerBufferInfo() = {0, 0}; auto result = channel.doTransition( [&]() { - if (*channel.fromWorkerBufferIndex() != 1) { + if (channel.fromWorkerBufferInfo()->index_ != 1) { + throw cms::Exception("BadValue") << "wrong index value of fromWorkerBufferInfo " + << static_cast(channel.fromWorkerBufferInfo()->index_); + } + if (channel.fromWorkerBufferInfo()->identifier_ != 1) { throw cms::Exception("BadValue") - << "wrong value of fromWorkerBufferIndex " << *channel.fromWorkerBufferIndex(); + << "wrong identifier value of fromWorkerBufferInfo " << channel.fromWorkerBufferInfo()->identifier_; } if (not channel.shouldKeepEvent()) { throw cms::Exception("BadValue") << "told not to keep event"; @@ -56,12 +60,16 @@ namespace { } } { - *channel.toWorkerBufferIndex() = 1; + *channel.toWorkerBufferInfo() = {1, 1}; auto result = channel.doTransition( [&]() { - if (*channel.fromWorkerBufferIndex() != 0) { + if (channel.fromWorkerBufferInfo()->index_ != 0) { + throw cms::Exception("BadValue") << "wrong index value of fromWorkerBufferInfo " + << static_cast(channel.fromWorkerBufferInfo()->index_); + } + if (channel.fromWorkerBufferInfo()->identifier_ != 2) { throw cms::Exception("BadValue") - << "wrong value of fromWorkerBufferIndex " << *channel.fromWorkerBufferIndex(); + << "wrong identifier value of fromWorkerBufferInfo " << channel.fromWorkerBufferInfo()->identifier_; } if (channel.shouldKeepEvent()) { throw cms::Exception("BadValue") << "told to keep event"; @@ -105,10 +113,15 @@ namespace { throw cms::Exception("BadValue") << "wrong transitionID received " << static_cast(iTransitionID); } - if (*channel.toWorkerBufferIndex() != 0) { - throw cms::Exception("BadValue") << "wrong toWorkerBufferIndex received " << *channel.toWorkerBufferIndex(); + if (channel.toWorkerBufferInfo()->index_ != 0) { + throw cms::Exception("BadValue") + << "wrong toWorkerBufferInfo index received " << static_cast(channel.toWorkerBufferInfo()->index_); } - *channel.fromWorkerBufferIndex() = 1; + if (channel.toWorkerBufferInfo()->identifier_ != 0) { + throw cms::Exception("BadValue") + << "wrong toWorkerBufferInfo identifier received " << channel.toWorkerBufferInfo()->identifier_; + } + *channel.fromWorkerBufferInfo() = {1, 1}; channel.shouldKeepEvent(true); break; } @@ -121,10 +134,15 @@ namespace { throw cms::Exception("BadValue") << "wrong transitionID received " << static_cast(iTransitionID); } - if (*channel.toWorkerBufferIndex() != 1) { - throw cms::Exception("BadValue") << "wrong toWorkerBufferIndex received " << *channel.toWorkerBufferIndex(); + if (channel.toWorkerBufferInfo()->index_ != 1) { + throw cms::Exception("BadValue") + << "wrong toWorkerBufferInfo index received " << static_cast(channel.toWorkerBufferInfo()->index_); + } + if (channel.toWorkerBufferInfo()->identifier_ != 1) { + throw cms::Exception("BadValue") + << "wrong toWorkerBufferInfo identifier received " << channel.toWorkerBufferInfo()->identifier_; } - *channel.fromWorkerBufferIndex() = 0; + *channel.fromWorkerBufferInfo() = {2, 0}; channel.shouldKeepEvent(false); break; } diff --git a/GeneratorInterface/Core/bin/BuildFile.xml b/GeneratorInterface/Core/bin/BuildFile.xml new file mode 100644 index 0000000000000..97641bb91eec0 --- /dev/null +++ b/GeneratorInterface/Core/bin/BuildFile.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/GeneratorInterface/Core/bin/externalGenerator.cc b/GeneratorInterface/Core/bin/externalGenerator.cc new file mode 100644 index 0000000000000..60a519e00c094 --- /dev/null +++ b/GeneratorInterface/Core/bin/externalGenerator.cc @@ -0,0 +1,274 @@ +#include "boost/program_options.hpp" + +#include +#include +#include +#include +#include + +#include "FWCore/TestProcessor/interface/TestProcessor.h" + +#include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenRunInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h" + +#include "FWCore/Services/interface/ExternalRandomNumberGeneratorService.h" + +#include "FWCore/SharedMemory/interface/WriteBuffer.h" +#include "FWCore/SharedMemory/interface/ReadBuffer.h" +#include "FWCore/SharedMemory/interface/WorkerChannel.h" +#include "FWCore/SharedMemory/interface/ROOTSerializer.h" +#include "FWCore/SharedMemory/interface/ROOTDeserializer.h" +#include "FWCore/SharedMemory/interface/WorkerMonitorThread.h" + +static char const* const kMemoryNameOpt = "memory-name"; +static char const* const kMemoryNameCommandOpt = "memory-name,m"; +static char const* const kUniqueIDOpt = "unique-id"; +static char const* const kUniqueIDCommandOpt = "unique-id,i"; +static char const* const kHelpOpt = "help"; +static char const* const kHelpCommandOpt = "help,h"; +static char const* const kVerboseOpt = "verbose"; +static char const* const kVerboseCommandOpt = "verbose,v"; + +//NOTE: Can use TestProcessor as the harness for the worker + +using namespace edm::shared_memory; +class Harness { +public: + Harness(std::string const& iConfig, edm::ServiceToken iToken) + : tester_(edm::test::TestProcessor::Config{iConfig}, iToken) {} + + ExternalGeneratorLumiInfo getBeginLumiValue(unsigned int iLumi) { + auto lumi = tester_.testBeginLuminosityBlock(iLumi); + ExternalGeneratorLumiInfo returnValue; + returnValue.header_ = *lumi.get(); + return returnValue; + } + + ExternalGeneratorEventInfo getEventValue() { + ExternalGeneratorEventInfo returnValue; + auto event = tester_.test(); + returnValue.hepmc_ = *event.get("unsmeared"); + returnValue.eventInfo_ = *event.get(); + returnValue.keepEvent_ = event.modulePassed(); + return returnValue; + } + + GenLumiInfoProduct getEndLumiValue() { + auto lumi = tester_.testEndLuminosityBlock(); + return *lumi.get(); + } + + GenRunInfoProduct getEndRunValue() { + auto run = tester_.testEndRun(); + return *run.get(); + } + +private: + edm::test::TestProcessor tester_; +}; + +template +using Serializer = ROOTSerializer; + +int main(int argc, char* argv[]) { + std::string descString(argv[0]); + descString += " [--"; + descString += kMemoryNameOpt; + descString += "] memory_name"; + boost::program_options::options_description desc(descString); + + desc.add_options()(kHelpCommandOpt, "produce help message")( + kMemoryNameCommandOpt, boost::program_options::value(), "memory name")( + kUniqueIDCommandOpt, boost::program_options::value(), "unique id")(kVerboseCommandOpt, + "verbose output"); + + boost::program_options::positional_options_description p; + p.add(kMemoryNameOpt, 1); + p.add(kUniqueIDOpt, 2); + + boost::program_options::options_description all_options("All Options"); + all_options.add(desc); + + boost::program_options::variables_map vm; + try { + store(boost::program_options::command_line_parser(argc, argv).options(all_options).positional(p).run(), vm); + notify(vm); + } catch (boost::program_options::error const& iException) { + std::cout << argv[0] << ": Error while trying to process command line arguments:\n" + << iException.what() << "\nFor usage and an options list, please do 'cmsRun --help'."; + return 1; + } + + if (vm.count(kHelpOpt)) { + std::cout << desc << std::endl; + return 0; + } + + bool verbose = false; + if (vm.count(kVerboseOpt)) { + verbose = true; + } + + if (!vm.count(kMemoryNameOpt)) { + std::cout << " no argument given" << std::endl; + return 1; + } + + if (!vm.count(kUniqueIDOpt)) { + std::cout << " no second argument given" << std::endl; + return 1; + } + + WorkerMonitorThread monitorThread; + + monitorThread.startThread(); + + CMS_SA_ALLOW try { + std::string const memoryName(vm[kMemoryNameOpt].as()); + std::string const uniqueID(vm[kUniqueIDOpt].as()); + { + //This class is holding the lock + WorkerChannel communicationChannel(memoryName, uniqueID); + + WriteBuffer sm_buffer{memoryName, communicationChannel.fromWorkerBufferInfo()}; + ReadBuffer sm_readbuffer{std::string("Rand") + memoryName, communicationChannel.toWorkerBufferInfo()}; + int counter = 0; + + //The lock must be released if there is a catastrophic signal + auto lockPtr = communicationChannel.accessLock(); + monitorThread.setAction([lockPtr]() { + if (lockPtr) { + std::cerr << "SIGNAL CAUGHT: unlock\n"; + lockPtr->unlock(); + } + }); + + Serializer serializer(sm_buffer); + Serializer bl_serializer(sm_buffer); + Serializer el_serializer(sm_buffer); + Serializer er_serializer(sm_buffer); + + ROOTDeserializer random_deserializer(sm_readbuffer); + + std::cerr << uniqueID << " process: initializing " << std::endl; + int nlines; + std::cin >> nlines; + + std::string configuration; + for (int i = 0; i < nlines; ++i) { + std::string c; + std::getline(std::cin, c); + if (verbose) { + std::cerr << c << "\n"; + } + configuration += c + "\n"; + } + + edm::ExternalRandomNumberGeneratorService* randomService = new edm::ExternalRandomNumberGeneratorService; + auto serviceToken = + edm::ServiceRegistry::createContaining(std::unique_ptr(randomService)); + Harness harness(configuration, serviceToken); + + //Either ROOT or the Framework are overriding the signal handlers + monitorThread.setupSignalHandling(); + + if (verbose) { + std::cerr << uniqueID << " process: done initializing" << std::endl; + } + communicationChannel.workerSetupDone(); + + if (verbose) + std::cerr << uniqueID << " process: waiting " << counter << std::endl; + communicationChannel.handleTransitions([&](edm::Transition iTransition, unsigned long long iTransitionID) { + ++counter; + switch (iTransition) { + case edm::Transition::BeginRun: { + if (verbose) + std::cerr << uniqueID << " process: start beginRun " << std::endl; + if (verbose) + std::cerr << uniqueID << " process: end beginRun " << std::endl; + + break; + } + case edm::Transition::BeginLuminosityBlock: { + if (verbose) + std::cerr << uniqueID << " process: start beginLumi " << std::endl; + auto randState = random_deserializer.deserialize(); + if (verbose) + std::cerr << uniqueID << " random " << randState.state_.size() << " " << randState.seed_ << std::endl; + randomService->setState(randState.state_, randState.seed_); + auto value = harness.getBeginLumiValue(iTransitionID); + value.randomState_.state_ = randomService->getState(); + value.randomState_.seed_ = randomService->mySeed(); + + bl_serializer.serialize(value); + if (verbose) + std::cerr << uniqueID << " process: end beginLumi " << std::endl; + if (verbose) + std::cerr << uniqueID << " rand " << value.randomState_.state_.size() << " " << value.randomState_.seed_ + << std::endl; + break; + } + case edm::Transition::Event: { + if (verbose) + std::cerr << uniqueID << " process: event " << counter << std::endl; + auto randState = random_deserializer.deserialize(); + randomService->setState(randState.state_, randState.seed_); + auto value = harness.getEventValue(); + value.randomState_.state_ = randomService->getState(); + value.randomState_.seed_ = randomService->mySeed(); + + if (verbose) + std::cerr << uniqueID << " process: event " << counter << std::endl; + + serializer.serialize(value); + if (verbose) + std::cerr << uniqueID << " process: " + << " " << counter << std::endl; + //usleep(10000000); + break; + } + case edm::Transition::EndLuminosityBlock: { + if (verbose) + std::cerr << uniqueID << " process: start endLumi " << std::endl; + auto value = harness.getEndLumiValue(); + + el_serializer.serialize(value); + if (verbose) + std::cerr << uniqueID << " process: end endLumi " << std::endl; + + break; + } + case edm::Transition::EndRun: { + if (verbose) + std::cerr << uniqueID << " process: start endRun " << std::endl; + auto value = harness.getEndRunValue(); + + er_serializer.serialize(value); + if (verbose) + std::cerr << uniqueID << " process: end endRun " << std::endl; + + break; + } + default: { + assert(false); + } + } + if (verbose) + std::cerr << uniqueID << " process: notifying and waiting " << counter << std::endl; + }); + } + } catch (std::exception const& iExcept) { + std::cerr << "caught exception \n" << iExcept.what() << "\n"; + return 1; + } catch (...) { + std::cerr << "caught unknown exception"; + return 1; + } + return 0; +} diff --git a/GeneratorInterface/Core/plugins/BuildFile.xml b/GeneratorInterface/Core/plugins/BuildFile.xml index af12a8119d35c..a119db8a7e1f2 100644 --- a/GeneratorInterface/Core/plugins/BuildFile.xml +++ b/GeneratorInterface/Core/plugins/BuildFile.xml @@ -4,6 +4,7 @@ + diff --git a/GeneratorInterface/Core/plugins/CompareGeneratorResultsAnalyzer.cc b/GeneratorInterface/Core/plugins/CompareGeneratorResultsAnalyzer.cc new file mode 100644 index 0000000000000..e5aa49ee96fee --- /dev/null +++ b/GeneratorInterface/Core/plugins/CompareGeneratorResultsAnalyzer.cc @@ -0,0 +1,361 @@ +#include "FWCore/Framework/interface/global/EDAnalyzer.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/Run.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +#include "FWCore/Utilities/interface/Exception.h" + +#include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenRunInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" + +#include +#include + +namespace cgra { + struct DummyCache {}; +}; // namespace cgra + +class CompareGeneratorResultsAnalyzer + : public edm::global::EDAnalyzer, edm::LuminosityBlockCache> { +public: + CompareGeneratorResultsAnalyzer(edm::ParameterSet const&); + + std::shared_ptr globalBeginRun(edm::Run const&, edm::EventSetup const&) const override; + void globalEndRun(edm::Run const&, edm::EventSetup const&) const override; + + std::shared_ptr globalBeginLuminosityBlock(edm::LuminosityBlock const&, + edm::EventSetup const&) const override; + void globalEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) const override; + + void analyze(edm::StreamID, edm::Event const&, edm::EventSetup const&) const override; + +private: + std::string mod1_; + std::string mod2_; + + edm::EDGetTokenT evToken1_; + edm::EDGetTokenT evToken2_; + + edm::EDGetTokenT hepMCToken1_; + edm::EDGetTokenT hepMCToken2_; + + edm::EDGetTokenT lumiHeaderToken1_; + edm::EDGetTokenT lumiHeaderToken2_; + + edm::EDGetTokenT lumiProductToken1_; + edm::EDGetTokenT lumiProductToken2_; + + edm::EDGetTokenT runProductToken1_; + edm::EDGetTokenT runProductToken2_; + + bool allowXSecDifferences_; +}; + +CompareGeneratorResultsAnalyzer::CompareGeneratorResultsAnalyzer(edm::ParameterSet const& iPSet) + : mod1_{iPSet.getUntrackedParameter("module1")}, + mod2_{iPSet.getUntrackedParameter("module2")}, + evToken1_{consumes(mod1_)}, + evToken2_{consumes(mod2_)}, + hepMCToken1_{consumes(edm::InputTag(mod1_, "unsmeared"))}, + hepMCToken2_{consumes(edm::InputTag(mod2_, "unsmeared"))}, + lumiHeaderToken1_{consumes(mod1_)}, + lumiHeaderToken2_{consumes(mod2_)}, + lumiProductToken1_{consumes(mod1_)}, + lumiProductToken2_{consumes(mod2_)}, + runProductToken1_{consumes(mod1_)}, + runProductToken2_{consumes(mod2_)}, + allowXSecDifferences_{iPSet.getUntrackedParameter("allowXSecDifferences", false)} {} + +std::shared_ptr CompareGeneratorResultsAnalyzer::globalBeginRun(edm::Run const&, + edm::EventSetup const&) const { + return std::shared_ptr(); +} + +void CompareGeneratorResultsAnalyzer::globalEndRun(edm::Run const& iRun, edm::EventSetup const&) const { + auto const& prod1 = iRun.get(runProductToken1_); + auto const& prod2 = iRun.get(runProductToken2_); + + if (not prod1.isProductEqual(prod2)) { + throw cms::Exception("ComparisonFailure") << "The GenRunInfoProducts are different"; + } +} + +std::shared_ptr CompareGeneratorResultsAnalyzer::globalBeginLuminosityBlock( + edm::LuminosityBlock const& iLumi, edm::EventSetup const&) const { + auto const& prod1 = iLumi.get(lumiHeaderToken1_); + auto const& prod2 = iLumi.get(lumiHeaderToken2_); + + if (prod1.randomConfigIndex() != prod2.randomConfigIndex()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoHeaders have different randomConfigIndex " + << prod1.randomConfigIndex() << " " << prod2.randomConfigIndex(); + } + + if (prod1.configDescription() != prod2.configDescription()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoHeaders have different configDescription " + << prod1.configDescription() << " " << prod2.configDescription(); + } + + if (prod1.lheHeaders().size() != prod2.lheHeaders().size()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoHeaders have different lheHeaders " + << prod1.lheHeaders().size() << " " << prod2.lheHeaders().size(); + } + + if (prod1.weightNames().size() != prod2.weightNames().size()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoHeaders have different weightNames " + << prod1.weightNames().size() << " " << prod2.weightNames().size(); + } + + return std::shared_ptr(); +} + +namespace { + void compare(size_t iIndex, + GenLumiInfoProduct::ProcessInfo const& p1, + GenLumiInfoProduct::ProcessInfo const& p2, + bool allowXSecDifferences) { + if (p1.process() != p2.process()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] process " << p1.process() << " " << p2.process(); + } + + if (p1.nPassPos() != p2.nPassPos()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] nPassPos " << p1.nPassPos() << " " << p2.nPassPos(); + } + + if (p1.nPassNeg() != p2.nPassNeg()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] nPassNeg " << p1.nPassNeg() << " " << p2.nPassNeg(); + } + + if (p1.nTotalPos() != p2.nTotalPos()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] nTotalPos " << p1.nTotalPos() << " " << p2.nTotalPos(); + } + + if (p1.nTotalNeg() != p2.nTotalNeg()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] nTotalNeg " << p1.nTotalNeg() << " " << p2.nTotalNeg(); + } + + if (p1.lheXSec().error() != p2.lheXSec().error()) { + if (allowXSecDifferences) { + edm::LogWarning("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] lheXSec.error " + << p1.lheXSec().error() << " " << p2.lheXSec().error(); + } else { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] lheXSec.error " + << p1.lheXSec().error() << " " << p2.lheXSec().error(); + } + } + + if (p1.lheXSec().value() != p2.lheXSec().value()) { + if (allowXSecDifferences) { + //throw cms::Exception("ComparisonFailure") + edm::LogWarning("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] lheXSec.value " + << p1.lheXSec().value() << " " << p2.lheXSec().value(); + } else { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] lheXSec.value " + << p1.lheXSec().value() << " " << p2.lheXSec().value(); + } + } + + if (p1.tried().n() != p2.tried().n()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] tried.n " << p1.tried().n() << " " << p2.tried().n(); + } + + if (p1.tried().sum() != p2.tried().sum()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] tried.sum " << p1.tried().sum() << " " << p2.tried().sum(); + } + + if (p1.tried().sum2() != p2.tried().sum2()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] tried.sum2 " << p1.tried().sum2() << " " << p2.tried().sum2(); + } + + if (p1.selected().n() != p2.selected().n()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] selected.n " << p1.selected().n() << " " << p2.selected().n(); + } + + if (p1.selected().sum() != p2.selected().sum()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] selected.sum " + << p1.selected().sum() << " " << p2.selected().sum(); + } + + if (p1.selected().sum2() != p2.selected().sum2()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] selected.sum2 " + << p1.selected().sum2() << " " << p2.selected().sum2(); + } + + if (p1.killed().n() != p2.killed().n()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] killed.n " << p1.killed().n() << " " << p2.killed().n(); + } + + if (p1.killed().sum() != p2.killed().sum()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] killed sum " << p1.killed().sum() << " " << p2.killed().sum(); + } + + if (p1.killed().sum2() != p2.killed().sum2()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] killed.sum2 " << p1.killed().sum2() << " " << p2.killed().sum2(); + } + + if (p1.accepted().n() != p2.accepted().n()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex + << "] accepted.n " << p1.accepted().n() << " " << p2.accepted().n(); + } + + if (p1.accepted().sum() != p2.accepted().sum()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] accepted.sum " + << p1.accepted().sum() << " " << p2.accepted().sum(); + } + + if (p1.accepted().sum2() != p2.accepted().sum2()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] accepted.sum2 " + << p1.accepted().sum2() << " " << p2.accepted().sum2(); + } + + if (p1.acceptedBr().n() != p2.acceptedBr().n()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] acceptedBr.n " + << p1.acceptedBr().n() << " " << p2.acceptedBr().n(); + } + + if (p1.acceptedBr().sum() != p2.acceptedBr().sum()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] acceptedZBr.sum " + << p1.acceptedBr().sum() << " " << p2.acceptedBr().sum(); + } + + if (p1.acceptedBr().sum2() != p2.acceptedBr().sum2()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoProducts have different getProcessInfos()[" << iIndex << "] acceptedBr.sum2 " + << p1.acceptedBr().sum2() << " " << p2.acceptedBr().sum2(); + } + } +} // namespace + +void CompareGeneratorResultsAnalyzer::globalEndLuminosityBlock(edm::LuminosityBlock const& iLumi, + edm::EventSetup const&) const { + auto const& prod1 = iLumi.get(lumiProductToken1_); + auto const& prod2 = iLumi.get(lumiProductToken2_); + + if (not prod1.isProductEqual(prod2)) { + if (prod1.getHEPIDWTUP() != prod1.getHEPIDWTUP()) { + throw cms::Exception("ComparisonFailure") << "The GenLumiInfoProducts have different getHEPIDWTUP " + << prod1.getHEPIDWTUP() << " " << prod2.getHEPIDWTUP(); + } + + if (prod1.getProcessInfos().size() != prod2.getProcessInfos().size()) { + throw cms::Exception("ComparisonFailure") + << "The GenLumiInfoHeaders have different getProcessInfos " << prod1.getProcessInfos().size() << " " + << prod2.getProcessInfos().size(); + } + + for (size_t i = 0; i < prod1.getProcessInfos().size(); ++i) { + compare(i, prod1.getProcessInfos()[i], prod2.getProcessInfos()[i], allowXSecDifferences_); + } + + if (not allowXSecDifferences_) { + throw cms::Exception("ComparisionFailure") << "The GenLumiInfoProducts are different"; + } + } +} + +namespace { + void compare(GenEventInfoProduct const& prod1, GenEventInfoProduct const& prod2) { + if (prod1.weights().size() != prod2.weights().size()) { + throw cms::Exception("ComparisonFailure") << "The GenEventInfoProducts have different weights " + << prod1.weights().size() << " " << prod2.weights().size(); + } + + if (prod1.binningValues().size() != prod2.binningValues().size()) { + throw cms::Exception("ComparisonFailure") << "The GenEventInfoProducts have different binningValues " + << prod1.binningValues().size() << " " << prod2.binningValues().size(); + } + + if (prod1.DJRValues().size() != prod2.DJRValues().size()) { + throw cms::Exception("ComparisonFailure") << "The GenEventInfoProducts have different DJRValues " + << prod1.DJRValues().size() << " " << prod2.DJRValues().size(); + } + + if (prod1.signalProcessID() != prod2.signalProcessID()) { + throw cms::Exception("ComparisonFailure") << "The GenEventInfoProducts have different signalProcessID " + << prod1.signalProcessID() << " " << prod2.signalProcessID(); + } + + if (prod1.qScale() != prod2.qScale()) { + throw cms::Exception("ComparisonFailure") + << "The GenEventInfoProducts have different qScale " << prod1.qScale() << " " << prod2.qScale(); + } + + if (prod1.alphaQCD() != prod2.alphaQCD()) { + throw cms::Exception("ComparisonFailure") + << "The GenEventInfoProducts have different alphaQCD " << prod1.alphaQCD() << " " << prod2.alphaQCD(); + } + + if (prod1.alphaQED() != prod2.alphaQED()) { + throw cms::Exception("ComparisonFailure") + << "The GenEventInfoProducts have different alphaQED " << prod1.alphaQED() << " " << prod2.alphaQED(); + } + + if (prod1.nMEPartons() != prod2.nMEPartons()) { + throw cms::Exception("ComparisonFailure") + << "The GenEventInfoProducts have different nMEPartons " << prod1.nMEPartons() << " " << prod2.nMEPartons(); + } + + if (prod1.nMEPartonsFiltered() != prod2.nMEPartonsFiltered()) { + throw cms::Exception("ComparisonFailure") << "The GenEventInfoProducts have different nMEPartonsFiltered " + << prod1.nMEPartonsFiltered() << " " << prod2.nMEPartonsFiltered(); + } + } + + void compare(HepMC::GenEvent const& prod1, HepMC::GenEvent const& prod2) { + if (prod1.signal_process_id() != prod2.signal_process_id()) { + throw cms::Exception("ComparisonFailure") << "The HepMCProducts have different signal_process_id " + << prod1.signal_process_id() << " " << prod2.signal_process_id(); + } + + if (prod1.vertices_size() != prod2.vertices_size()) { + throw cms::Exception("ComparisonFailure") << "The HepMCProducts have different vertices_size() " + << prod1.vertices_size() << " " << prod2.vertices_size(); + } + + if (prod1.particles_size() != prod2.particles_size()) { + throw cms::Exception("ComparisonFailure") << "The HepMCProducts have different particles_size() " + << prod1.particles_size() << " " << prod2.particles_size(); + } + } +} // namespace + +void CompareGeneratorResultsAnalyzer::analyze(edm::StreamID, edm::Event const& iEvent, edm::EventSetup const&) const { + auto const& prod1 = iEvent.get(evToken1_); + auto const& prod2 = iEvent.get(evToken2_); + + compare(prod1, prod2); + + auto const& hepmc1 = iEvent.get(hepMCToken1_); + auto const& hepmc2 = iEvent.get(hepMCToken2_); + + compare(hepmc1.getHepMCData(), hepmc2.getHepMCData()); +} + +DEFINE_FWK_MODULE(CompareGeneratorResultsAnalyzer); diff --git a/GeneratorInterface/Core/plugins/ExternalGeneratorFilter.cc b/GeneratorInterface/Core/plugins/ExternalGeneratorFilter.cc new file mode 100644 index 0000000000000..91eb2c1dac81c --- /dev/null +++ b/GeneratorInterface/Core/plugins/ExternalGeneratorFilter.cc @@ -0,0 +1,348 @@ +#include "FWCore/Framework/interface/global/EDFilter.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Utilities/interface/EDPutToken.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/Utilities/interface/RandomNumberGenerator.h" + +#include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenRunInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h" +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h" + +#include "FWCore/SharedMemory/interface/ReadBuffer.h" +#include "FWCore/SharedMemory/interface/WriteBuffer.h" +#include "FWCore/SharedMemory/interface/ControllerChannel.h" +#include "FWCore/SharedMemory/interface/ROOTDeserializer.h" +#include "FWCore/SharedMemory/interface/ROOTSerializer.h" + +#include "CLHEP/Random/RandomEngine.h" +#include "CLHEP/Random/engineIDulong.h" +#include "CLHEP/Random/RanecuEngine.h" + +#include +#include + +using namespace edm::shared_memory; +namespace externalgen { + + struct StreamCache { + StreamCache(const std::string& iConfig, int id, bool verbose) + : id_{id}, + channel_("extGen", id_), + readBuffer_{channel_.sharedMemoryName(), channel_.fromWorkerBufferInfo()}, + writeBuffer_{std::string("Rand") + channel_.sharedMemoryName(), channel_.toWorkerBufferInfo()}, + deserializer_{readBuffer_}, + er_deserializer_{readBuffer_}, + bl_deserializer_{readBuffer_}, + el_deserializer_(readBuffer_), + randSerializer_(writeBuffer_) { + //make sure output is flushed before popen does any writing + fflush(stdout); + fflush(stderr); + + channel_.setupWorker([&]() { + using namespace std::string_literals; + edm::LogSystem("ExternalProcess") << id_ << " starting external process \n"; + std::string verboseCommand; + if (verbose) { + verboseCommand = "--verbose "; + } + pipe_ = + popen(("cmsExternalGenerator "s + verboseCommand + channel_.sharedMemoryName() + " " + channel_.uniqueID()) + .c_str(), + "w"); + + if (nullptr == pipe_) { + abort(); + } + + { + auto nlines = std::to_string(std::count(iConfig.begin(), iConfig.end(), '\n')); + auto result = fwrite(nlines.data(), sizeof(char), nlines.size(), pipe_); + assert(result = nlines.size()); + result = fwrite(iConfig.data(), sizeof(char), iConfig.size(), pipe_); + assert(result == iConfig.size()); + fflush(pipe_); + } + }); + } + + template + auto doTransition(SERIAL& iDeserializer, edm::Transition iTrans, unsigned long long iTransitionID) + -> decltype(iDeserializer.deserialize()) { + decltype(iDeserializer.deserialize()) value; + if (not channel_.doTransition( + [&value, &iDeserializer]() { value = iDeserializer.deserialize(); }, iTrans, iTransitionID)) { + externalFailed_ = true; + throw cms::Exception("ExternalFailed") << "failed waiting for external process"; + } + return value; + } + ExternalGeneratorEventInfo produce(edm::StreamID iStream, unsigned long long iTransitionID) { + edm::Service gen; + auto& engine = gen->getEngine(iStream); + edm::RandomNumberGeneratorState state{engine.put(), engine.getSeed()}; + randSerializer_.serialize(state); + + return doTransition(deserializer_, edm::Transition::Event, iTransitionID); + } + + std::optional endRunProduce(unsigned long long iTransitionID) { + if (not externalFailed_) { + return doTransition(er_deserializer_, edm::Transition::EndRun, iTransitionID); + } + return {}; + } + + ExternalGeneratorLumiInfo beginLumiProduce(unsigned long long iTransitionID, + edm::RandomNumberGeneratorState const& iState) { + //NOTE: root serialize requires a `void*` not a `void const*` even though it doesn't modify the object + randSerializer_.serialize(const_cast(iState)); + return doTransition(bl_deserializer_, edm::Transition::BeginLuminosityBlock, iTransitionID); + } + + std::optional endLumiProduce(unsigned long long iTransitionID) { + if (not externalFailed_) { + return doTransition(el_deserializer_, edm::Transition::EndLuminosityBlock, iTransitionID); + } + return {}; + } + + ~StreamCache() { + channel_.stopWorker(); + pclose(pipe_); + } + + private: + std::string unique_name(std::string iBase) { + auto pid = getpid(); + iBase += std::to_string(pid); + iBase += "_"; + iBase += std::to_string(id_); + + return iBase; + } + + int id_; + FILE* pipe_; + ControllerChannel channel_; + ReadBuffer readBuffer_; + WriteBuffer writeBuffer_; + + template + using Deserializer = ROOTDeserializer; + Deserializer deserializer_; + Deserializer er_deserializer_; + Deserializer bl_deserializer_; + Deserializer el_deserializer_; + ROOTSerializer randSerializer_; + + bool externalFailed_ = false; + }; + + struct RunCache { + //Only stream 0 sets this at stream end Run and it is read at global end run + // the framework guarantees those calls can not happen simultaneously + CMS_THREAD_SAFE mutable GenRunInfoProduct runInfo_; + }; + struct LumiCache { + LumiCache(std::vector iState, long iSeed) : randomState_(std::move(iState), iSeed) {} + //Only stream 0 sets this at stream end Lumi and it is read at global end Lumi + // the framework guarantees those calls can not happen simultaneously + CMS_THREAD_SAFE mutable edm::RandomNumberGeneratorState randomState_; + }; +} // namespace externalgen + +class ExternalGeneratorFilter : public edm::global::EDFilter, + edm::RunCache, + edm::EndRunProducer, + edm::LuminosityBlockCache, + edm::LuminosityBlockSummaryCache, + edm::BeginLuminosityBlockProducer, + edm::EndLuminosityBlockProducer> { +public: + ExternalGeneratorFilter(edm::ParameterSet const&); + + std::unique_ptr beginStream(edm::StreamID) const final; + bool filter(edm::StreamID, edm::Event&, edm::EventSetup const&) const final; + + std::shared_ptr globalBeginRun(edm::Run const&, edm::EventSetup const&) const final; + void streamBeginRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const final; + void streamEndRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const final; + void globalEndRun(edm::Run const&, edm::EventSetup const&) const final {} + void globalEndRunProduce(edm::Run&, edm::EventSetup const&) const final; + + void globalBeginLuminosityBlockProduce(edm::LuminosityBlock&, edm::EventSetup const&) const final; + std::shared_ptr globalBeginLuminosityBlock(edm::LuminosityBlock const&, + edm::EventSetup const&) const final; + std::shared_ptr globalBeginLuminosityBlockSummary(edm::LuminosityBlock const&, + edm::EventSetup const&) const final; + void streamBeginLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const final; + void streamEndLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const final; + void streamEndLuminosityBlockSummary(edm::StreamID, + edm::LuminosityBlock const&, + edm::EventSetup const&, + GenLumiInfoProduct*) const final; + void globalEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) const final {} + void globalEndLuminosityBlockSummary(edm::LuminosityBlock const&, + edm::EventSetup const&, + GenLumiInfoProduct*) const final {} + void globalEndLuminosityBlockProduce(edm::LuminosityBlock&, + edm::EventSetup const&, + GenLumiInfoProduct const*) const final; + +private: + edm::EDPutTokenT const hepMCToken_; + edm::EDPutTokenT const genEventToken_; + edm::EDPutTokenT const runInfoToken_; + edm::EDPutTokenT const lumiHeaderToken_; + edm::EDPutTokenT const lumiInfoToken_; + + std::string const config_; + bool const verbose_; + + //This is set at beginStream and used for globalBeginRun + //The framework guarantees that non of those can happen concurrently + CMS_THREAD_SAFE mutable externalgen::StreamCache* stream0Cache_ = nullptr; + //A stream which has finished processing the last lumi is used for the + // call to globalBeginLuminosityBlockProduce + mutable std::atomic availableForBeginLumi_; + //Streams all see the lumis in the same order, we want to be sure to pick a stream cache + // to use at globalBeginLumi which just finished the most recent lumi and not a previous one + mutable std::atomic lastLumiIndex_ = 0; +}; + +ExternalGeneratorFilter::ExternalGeneratorFilter(edm::ParameterSet const& iPSet) + : hepMCToken_{produces("unsmeared")}, + genEventToken_{produces()}, + runInfoToken_{produces()}, + lumiHeaderToken_{produces()}, + lumiInfoToken_{produces()}, + config_{iPSet.getUntrackedParameter("@python_config")}, + verbose_{iPSet.getUntrackedParameter("_external_process_verbose_")} {} + +std::unique_ptr ExternalGeneratorFilter::beginStream(edm::StreamID iID) const { + auto const label = moduleDescription().moduleLabel(); + + using namespace std::string_literals; + + std::string config = R"_(from FWCore.TestProcessor.TestProcess import * +process = TestProcess() +)_"; + config += "process."s + label + "=" + config_ + "\n"; + config += "process.moduleToTest(process."s + label + ")\n"; + config += R"_( +process.add_(cms.Service("InitRootHandlers", UnloadRootSigHandler=cms.untracked.bool(True))) + )_"; + + auto cache = std::make_unique(config, iID.value(), verbose_); + if (iID.value() == 0) { + stream0Cache_ = cache.get(); + + availableForBeginLumi_ = stream0Cache_; + } + + return cache; +} + +bool ExternalGeneratorFilter::filter(edm::StreamID iID, edm::Event& iEvent, edm::EventSetup const&) const { + auto value = streamCache(iID)->produce(iID, iEvent.id().event()); + + edm::Service gen; + auto& engine = gen->getEngine(iID); + //if (value.randomState_.state_[0] != CLHEP::engineIDulong()) { + // engine.setSeed(value.randomState_.seed_, 0); + //} + engine.get(value.randomState_.state_); + + iEvent.emplace(hepMCToken_, std::move(value.hepmc_)); + iEvent.emplace(genEventToken_, std::move(value.eventInfo_)); + return value.keepEvent_; +} + +std::shared_ptr ExternalGeneratorFilter::globalBeginRun(edm::Run const&, + edm::EventSetup const&) const { + return std::make_shared(); +} + +void ExternalGeneratorFilter::streamBeginRun(edm::StreamID iID, edm::Run const& iRun, edm::EventSetup const&) const {} +void ExternalGeneratorFilter::streamEndRun(edm::StreamID iID, edm::Run const& iRun, edm::EventSetup const&) const { + if (iID.value() == 0) { + runCache(iRun.index())->runInfo_ = *streamCache(iID)->endRunProduce(iRun.run()); + } else { + (void)streamCache(iID)->endRunProduce(iRun.run()); + } +} +void ExternalGeneratorFilter::globalEndRunProduce(edm::Run& iRun, edm::EventSetup const&) const { + iRun.emplace(runInfoToken_, std::move(runCache(iRun.index())->runInfo_)); +} + +void ExternalGeneratorFilter::globalBeginLuminosityBlockProduce(edm::LuminosityBlock& iLuminosityBlock, + edm::EventSetup const&) const { + while (not availableForBeginLumi_.load()) { + } + + auto v = availableForBeginLumi_.load()->beginLumiProduce( + iLuminosityBlock.luminosityBlock(), luminosityBlockCache(iLuminosityBlock.index())->randomState_); + + edm::Service gen; + auto& engine = gen->getEngine(iLuminosityBlock.index()); + engine.get(v.randomState_.state_); + + iLuminosityBlock.emplace(lumiHeaderToken_, std::move(v.header_)); + + lastLumiIndex_.store(iLuminosityBlock.index()); +} + +std::shared_ptr ExternalGeneratorFilter::globalBeginLuminosityBlock( + edm::LuminosityBlock const& iLumi, edm::EventSetup const&) const { + edm::Service gen; + auto& engine = gen->getEngine(iLumi.index()); + auto s = engine.put(); + return std::make_shared(s, engine.getSeed()); +} + +std::shared_ptr ExternalGeneratorFilter::globalBeginLuminosityBlockSummary( + edm::LuminosityBlock const&, edm::EventSetup const&) const { + return std::make_shared(); +} + +void ExternalGeneratorFilter::streamBeginLuminosityBlock(edm::StreamID iID, + edm::LuminosityBlock const& iLuminosityBlock, + edm::EventSetup const&) const { + auto cache = streamCache(iID); + if (cache != availableForBeginLumi_.load()) { + (void)cache->beginLumiProduce(iLuminosityBlock.run(), luminosityBlockCache(iLuminosityBlock.index())->randomState_); + } else { + availableForBeginLumi_ = nullptr; + } +} + +void ExternalGeneratorFilter::streamEndLuminosityBlock(edm::StreamID iID, + edm::LuminosityBlock const& iLuminosityBlock, + edm::EventSetup const&) const {} + +void ExternalGeneratorFilter::streamEndLuminosityBlockSummary(edm::StreamID iID, + edm::LuminosityBlock const& iLuminosityBlock, + edm::EventSetup const&, + GenLumiInfoProduct* iProduct) const { + iProduct->mergeProduct(*streamCache(iID)->endLumiProduce(iLuminosityBlock.run())); + + if (lastLumiIndex_ == iLuminosityBlock.index()) { + externalgen::StreamCache* expected = nullptr; + + availableForBeginLumi_.compare_exchange_strong(expected, streamCache(iID)); + } +} + +void ExternalGeneratorFilter::globalEndLuminosityBlockProduce(edm::LuminosityBlock& iLuminosityBlock, + edm::EventSetup const&, + GenLumiInfoProduct const* iProduct) const { + iLuminosityBlock.emplace(lumiInfoToken_, std::move(*iProduct)); +} + +DEFINE_FWK_MODULE(ExternalGeneratorFilter); diff --git a/GeneratorInterface/Core/python/ExternalGeneratorFilter.py b/GeneratorInterface/Core/python/ExternalGeneratorFilter.py new file mode 100644 index 0000000000000..b7f2da8f41214 --- /dev/null +++ b/GeneratorInterface/Core/python/ExternalGeneratorFilter.py @@ -0,0 +1,43 @@ +import FWCore.ParameterSet.Config as cms + +class ExternalGeneratorFilter(cms.EDFilter): + def __init__(self, prod, _external_process_verbose_ = cms.untracked.bool(False)): + self.__dict__['_external_process_verbose_']=_external_process_verbose_ + self.__dict__['_prod'] = prod + super(cms.EDFilter,self).__init__('ExternalGeneratorFilter') + def __setattr__(self, name, value): + if name =='_external_process_verbose_': + return self.__dict__['_external_process_verbose_'] + setattr(self._prod, name, value) + def __getattr__(self, name): + if name =='_prod': + return self.__dict__['_prod'] + if name == '_external_process_verbose_': + return self.__dict__['_external_process_verbose_'] + return getattr(self._prod, name) + def clone(self, **params): + returnValue = ExternalGeneratorFilter.__new__(type(self)) + returnValue.__init__(self._prod.clone()) + return returnValue + def insertInto(self, parameterSet, myname): + newpset = parameterSet.newPSet() + newpset.addString(True, "@module_label", self.moduleLabel_(myname)) + newpset.addString(True, "@module_type", self.type_()) + newpset.addString(True, "@module_edm_type", cms.EDFilter.__name__) + newpset.addString(True, "@external_type", self._prod.type_()) + newpset.addString(False,"@python_config", self._prod.dumpPython()) + newpset.addBool(False,"_external_process_verbose_", self._external_process_verbose_.value()) + self._prod.insertContentsInto(newpset) + parameterSet.addPSet(True, self.nameInProcessDesc_(myname), newpset) + def dumpPython(self, options=cms.PrintOptions()): + cms.specialImportRegistry.registerUse(self) + result = "%s(" % self.__class__.__name__ # not including cms. since the deriving classes are not in cms "namespace" + options.indent() + result += "\n"+options.indentation() + self._prod.dumpPython(options) + result +=options.indentation()+",\n" + result += options.indentation() + self._external_process_verbose_.dumpPython(options) + options.unindent() + result += "\n)\n" + return result + +cms.specialImportRegistry.registerSpecialImportForType(ExternalGeneratorFilter, "from GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter") diff --git a/GeneratorInterface/Pythia8Interface/test/BuildFile.xml b/GeneratorInterface/Pythia8Interface/test/BuildFile.xml index 643d169882572..bdcda759de946 100755 --- a/GeneratorInterface/Pythia8Interface/test/BuildFile.xml +++ b/GeneratorInterface/Pythia8Interface/test/BuildFile.xml @@ -17,3 +17,7 @@ + + + + diff --git a/GeneratorInterface/Pythia8Interface/test/compare_external_generators_cfg.py b/GeneratorInterface/Pythia8Interface/test/compare_external_generators_cfg.py new file mode 100644 index 0000000000000..0168b7c86452d --- /dev/null +++ b/GeneratorInterface/Pythia8Interface/test/compare_external_generators_cfg.py @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("TEST") + +process.source =cms.Source("EmptySource") + +process.maxEvents.input = 10 + +process.load("Configuration.StandardSequences.SimulationRandomNumberGeneratorSeeds_cff") +process.RandomNumberGeneratorService.gen1 = process.RandomNumberGeneratorService.generator.clone() +process.RandomNumberGeneratorService.gen2 = process.RandomNumberGeneratorService.generator.clone() +process.gen1 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) + ) + +_pythia8 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) +) +from GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter +process.gen2 = ExternalGeneratorFilter(_pythia8) + +process.compare = cms.EDAnalyzer("CompareGeneratorResultsAnalyzer", module1 = cms.untracked.string("gen1"), module2 =cms.untracked.string("gen2")) + +process.p = cms.Path(process.gen1+process.gen2+process.compare) diff --git a/GeneratorInterface/Pythia8Interface/test/compare_external_generators_streams_cfg.py b/GeneratorInterface/Pythia8Interface/test/compare_external_generators_streams_cfg.py new file mode 100644 index 0000000000000..ecb96b74169af --- /dev/null +++ b/GeneratorInterface/Pythia8Interface/test/compare_external_generators_streams_cfg.py @@ -0,0 +1,40 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("TEST") + +process.source =cms.Source("EmptySource") + +process.maxEvents.input = 10 +process.options.numberOfStreams = 2 +process.options.numberOfThreads = 2 + +process.load("Configuration.StandardSequences.SimulationRandomNumberGeneratorSeeds_cff") +process.RandomNumberGeneratorService.gen1 = process.RandomNumberGeneratorService.generator.clone() +process.RandomNumberGeneratorService.gen2 = process.RandomNumberGeneratorService.generator.clone() +process.gen1 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) + ) + +_pythia8 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) +) +from GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter +process.gen2 = ExternalGeneratorFilter(_pythia8) + +process.sleeper = cms.EDProducer("timestudy::SleepingProducer", ivalue = cms.int32(1), consumes = cms.VInputTag(), eventTimes=cms.vdouble(1.)) + +process.compare = cms.EDAnalyzer("CompareGeneratorResultsAnalyzer", + module1 = cms.untracked.string("gen1"), + module2 =cms.untracked.string("gen2"), + allowXSecDifferences = cms.untracked.bool(True)) + +process.p = cms.Path(process.sleeper+process.gen1+process.gen2+process.compare) diff --git a/GeneratorInterface/Pythia8Interface/test/compare_identical_generators_cfg.py b/GeneratorInterface/Pythia8Interface/test/compare_identical_generators_cfg.py new file mode 100644 index 0000000000000..70735585c0192 --- /dev/null +++ b/GeneratorInterface/Pythia8Interface/test/compare_identical_generators_cfg.py @@ -0,0 +1,29 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("TEST") + +process.source =cms.Source("EmptySource") + +process.maxEvents.input = 10 + +process.load("Configuration.StandardSequences.SimulationRandomNumberGeneratorSeeds_cff") +process.RandomNumberGeneratorService.gen1 = process.RandomNumberGeneratorService.generator.clone() +process.RandomNumberGeneratorService.gen2 = process.RandomNumberGeneratorService.generator.clone() +process.gen1 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) + ) +process.gen2 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) + ) +process.compare = cms.EDAnalyzer("CompareGeneratorResultsAnalyzer", module1 = cms.untracked.string("gen1"), module2 =cms.untracked.string("gen2")) + +process.p = cms.Path(process.gen1+process.gen2+process.compare) diff --git a/GeneratorInterface/Pythia8Interface/test/test_catch2_External_Pythia8GeneratorFilter.cc b/GeneratorInterface/Pythia8Interface/test/test_catch2_External_Pythia8GeneratorFilter.cc new file mode 100644 index 0000000000000..af92c47204d71 --- /dev/null +++ b/GeneratorInterface/Pythia8Interface/test/test_catch2_External_Pythia8GeneratorFilter.cc @@ -0,0 +1,57 @@ +#include "catch.hpp" +#include "FWCore/TestProcessor/interface/TestProcessor.h" +#include "FWCore/Utilities/interface/Exception.h" + +static constexpr auto s_tag = "[External Pythia8GeneratorFilter]"; + +TEST_CASE("Standard checks of ExternalGeneratorFilter with Pythia8GeneratorFilter", s_tag) { + const std::string baseConfig{ + R"_(from FWCore.TestProcessor.TestProcess import * +process = TestProcess() +process.load("Configuration.StandardSequences.SimulationRandomNumberGeneratorSeeds_cff") +process.RandomNumberGeneratorService.toTest = process.RandomNumberGeneratorService.generator.clone() +from GeneratorInterface.Core.ExternalGeneratorFilter import ExternalGeneratorFilter +_pythia8 = cms.EDFilter("Pythia8GeneratorFilter", + comEnergy = cms.double(7000.), + PythiaParameters = cms.PSet( + pythia8_example02 = cms.vstring('HardQCD:all = on', + 'PhaseSpace:pTHatMin = 20.'), + parameterSets = cms.vstring('pythia8_example02') + ) +) +process.toTest = ExternalGeneratorFilter(_pythia8) + +process.add_(cms.Service('Tracer')) + +process.moduleToTest(process.toTest) +)_"}; + + edm::test::TestProcessor::Config config{baseConfig}; + SECTION("base configuration is OK") { REQUIRE_NOTHROW(edm::test::TestProcessor(config)); } + + SECTION("No event data") { + edm::test::TestProcessor tester(config); + + REQUIRE_NOTHROW(tester.test()); + } + + SECTION("beginJob and endJob only") { + edm::test::TestProcessor tester(config); + + REQUIRE_NOTHROW(tester.testBeginAndEndJobOnly()); + } + + SECTION("Run with no LuminosityBlocks") { + edm::test::TestProcessor tester(config); + + REQUIRE_NOTHROW(tester.testRunWithNoLuminosityBlocks()); + } + + SECTION("LuminosityBlock with no Events") { + edm::test::TestProcessor tester(config); + + REQUIRE_NOTHROW(tester.testLuminosityBlockWithNoEvents()); + } +} + +//Add additional TEST_CASEs to exercise the modules capabilities diff --git a/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h b/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h new file mode 100644 index 0000000000000..30d1a48b0fa49 --- /dev/null +++ b/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h @@ -0,0 +1,21 @@ +#ifndef SimDataFormats_GeneratorProducts_ExternalGeneratorEventInfo_h +#define SimDataFormats_GeneratorProducts_ExternalGeneratorEventInfo_h + +/** \class ExternalGeneratorEventInfo + * + * This class is an internal detail of the ExternalGeneratorFilter. It is the type + * used to transfer from the external process to ExternalGeneratorFilter the + * event information. + */ +#include "SimDataFormats/GeneratorProducts/interface/HepMCProduct.h" +#include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" +#include "DataFormats/Common/interface/RandomNumberGeneratorState.h" + +struct ExternalGeneratorEventInfo { + edm::HepMCProduct hepmc_; + GenEventInfoProduct eventInfo_; + edm::RandomNumberGeneratorState randomState_; + bool keepEvent_; +}; + +#endif diff --git a/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h b/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h new file mode 100644 index 0000000000000..479a35c4c84cb --- /dev/null +++ b/SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h @@ -0,0 +1,18 @@ +#ifndef SimDataFormats_GeneratorProducts_ExternalGeneratorLumiInfo_h +#define SimDataFormats_GeneratorProducts_ExternalGeneratorLumiInfo_h + +/** \class ExternalGeneratorLumiInfo + * + * This class is an internal detail of the ExternalGeneratorFilter. It is the type + * used to transfer from the external process to ExternalGeneratorFilter the + * begin LuminosityBlock information. + */ +#include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h" +#include "DataFormats/Common/interface/RandomNumberGeneratorState.h" + +struct ExternalGeneratorLumiInfo { + GenLumiInfoHeader header_; + edm::RandomNumberGeneratorState randomState_; +}; + +#endif diff --git a/SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h b/SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h index a8a7cd218f520..f48a377fc0588 100644 --- a/SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h +++ b/SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h @@ -2,6 +2,7 @@ #define SimDataFormats_GeneratorProducts_GenLumiInfoProduct_h #include +#include /** \class GenLumiInfoProduct * @@ -63,9 +64,12 @@ class GenLumiInfoProduct { double sum2() const { return sum2_; } void add(const FinalStat &other) { - n_ += other.n(); - sum_ += other.sum(); - sum2_ += other.sum2(); + //Hadronizers treat 0, -1., -1. to mean the value is not present + if (other.n() != 0 and other.sum_ != -1. and other.sum2() != -1.) { + n_ += other.n(); + sum_ += other.sum(); + sum2_ += other.sum2(); + } } bool operator==(const FinalStat &other) const { @@ -86,21 +90,22 @@ class GenLumiInfoProduct { // accessors int process() const { return process_; } - XSec lheXSec() const { return lheXSec_; } + XSec const &lheXSec() const { return lheXSec_; } unsigned int nPassPos() const { return nPassPos_; } unsigned int nPassNeg() const { return nPassNeg_; } unsigned int nTotalPos() const { return nTotalPos_; } unsigned int nTotalNeg() const { return nTotalNeg_; } - FinalStat tried() const { return tried_; } - FinalStat selected() const { return selected_; } - FinalStat killed() const { return killed_; } - FinalStat accepted() const { return accepted_; } - FinalStat acceptedBr() const { return acceptedBr_; } + FinalStat const &tried() const { return tried_; } + FinalStat const &selected() const { return selected_; } + FinalStat const &killed() const { return killed_; } + FinalStat const &accepted() const { return accepted_; } + FinalStat const &acceptedBr() const { return acceptedBr_; } // setters void addOthers(const ProcessInfo &other) { + mergeXSec(other.lheXSec(), other.selected().sum()); nPassPos_ += other.nPassPos(); nPassNeg_ += other.nPassNeg(); nTotalPos_ += other.nTotalPos(); @@ -124,6 +129,21 @@ class GenLumiInfoProduct { void setAcceptedBr(unsigned int n, double sum, double sum2) { acceptedBr_ = FinalStat(n, sum, sum2); } private: + void mergeXSec(XSec const &iXSec, double iWeight) { + if (iWeight <= 0.) { + return; + } + if (lheXSec_.value() <= 0.) { + lheXSec_ = iXSec; + } else { + bool useWeights = (lheXSec_.error() <= 0. || iXSec.error() <= 0.); + double wgt1 = useWeights ? selected().sum() : 1. / (lheXSec_.error() * lheXSec_.error()); + double wgt2 = useWeights ? iWeight : 1. / (iXSec.error() * iXSec.error()); + double xsec = (wgt1 * lheXSec_.value() + wgt2 * iXSec.value()) / (wgt1 + wgt2); + double err = useWeights ? 0. : 1.0 / std::sqrt(wgt1 + wgt2); + lheXSec_ = XSec(xsec, err); + } + } int process_; XSec lheXSec_; unsigned int nPassPos_; diff --git a/SimDataFormats/GeneratorProducts/src/GenLumiInfoProduct.cc b/SimDataFormats/GeneratorProducts/src/GenLumiInfoProduct.cc index 0ceb451c4c270..543a7b33d27ad 100644 --- a/SimDataFormats/GeneratorProducts/src/GenLumiInfoProduct.cc +++ b/SimDataFormats/GeneratorProducts/src/GenLumiInfoProduct.cc @@ -57,7 +57,7 @@ const bool operator==(const GenLumiInfoProduct& lhs, const GenLumiInfoProduct& r unsigned int rhssize = rhsVector.size(); bool condition = (lhs.getHEPIDWTUP() == rhs.getHEPIDWTUP()) && (lhssize == rhssize); - unsigned int passCounts = -999; + unsigned int passCounts = 0; if (condition) { for (unsigned int i = 0; i < lhssize; i++) { if (lhsVector[i] == rhsVector[i]) diff --git a/SimDataFormats/GeneratorProducts/src/classes.h b/SimDataFormats/GeneratorProducts/src/classes.h index 7039e45b3951c..f4ef658588213 100644 --- a/SimDataFormats/GeneratorProducts/src/classes.h +++ b/SimDataFormats/GeneratorProducts/src/classes.h @@ -17,6 +17,9 @@ #include "SimDataFormats/GeneratorProducts/interface/GenEventInfoProduct.h" #include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoProduct.h" #include "SimDataFormats/GeneratorProducts/interface/GenLumiInfoHeader.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorLumiInfo.h" +#include "SimDataFormats/GeneratorProducts/interface/ExternalGeneratorEventInfo.h" + #include //needed for backward compatibility between HepMC 2.06.xx and 2.05.yy diff --git a/SimDataFormats/GeneratorProducts/src/classes_def.xml b/SimDataFormats/GeneratorProducts/src/classes_def.xml index 05bf110a49686..665a75ab79e8f 100644 --- a/SimDataFormats/GeneratorProducts/src/classes_def.xml +++ b/SimDataFormats/GeneratorProducts/src/classes_def.xml @@ -241,4 +241,11 @@ ]]> + + + + + + +