Skip to content

Commit

Permalink
Merge pull request #34338 from civanch/oscar_to_event_setup
Browse files Browse the repository at this point in the history 
Use of ESGetToken in Oscar producer
  • Loading branch information
@cmsbuild
cmsbuild authored Jul 13, 2021
2 parents b5a6b22 + 3ee5f11 commit 639903d
Show file tree
Hide file tree
Showing 9 changed files with 88 additions and 127 deletions.
36 changes: 22 additions & 14 deletions SimG4Core/Application/interface/OscarMTMasterThread.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,22 @@
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/Framework/interface/ESWatcher.h"

#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"
#include "DetectorDescription/Core/interface/DDCompactView.h"
#include "DetectorDescription/DDCMS/interface/DDCompactView.h"
#include "Geometry/Records/interface/IdealGeometryRecord.h"

#include "HepPDT/ParticleDataTable.hh"
#include "SimGeneral/HepPDTRecord/interface/PDTRecord.h"

#include <memory>
#include <thread>
#include <mutex>
#include <condition_variable>

namespace edm {
class EventSetup;
}
class ConsumesCollector;
} // namespace edm

class RunManagerMT;

Expand All @@ -38,12 +43,12 @@ class OscarMTMasterThread {
void endRun() const;
void stopThread();

void callConsumes(edm::ConsumesCollector&& iC) const;

inline RunManagerMT& runManagerMaster() const { return *m_runManagerMaster; }
inline RunManagerMT* runManagerMasterPtr() const { return m_runManagerMaster.get(); }

private:
void readES(const edm::EventSetup& iSetup) const;

enum class ThreadState { NotExist = 0, BeginRun = 1, EndRun = 2, Destruct = 3 };

const bool m_pGeoFromDD4hep;
Expand All @@ -52,23 +57,26 @@ class OscarMTMasterThread {
std::thread m_masterThread;

// ES products needed for Geant4 initialization
mutable edm::ESWatcher<IdealGeometryRecord> idealGeomRcdWatcher_;
mutable edm::ESWatcher<IdealMagneticFieldRecord> idealMagRcdWatcher_;
mutable const DDCompactView* m_pDD;
mutable const cms::DDCompactView* m_pDD4hep;
mutable const HepPDT::ParticleDataTable* m_pTable;

mutable const DDCompactView* m_pDDD = nullptr;
mutable const cms::DDCompactView* m_pDD4Hep = nullptr;
mutable const HepPDT::ParticleDataTable* m_pTable = nullptr;
mutable edm::ESGetToken<DDCompactView, IdealGeometryRecord> m_DDD;
mutable edm::ESGetToken<cms::DDCompactView, IdealGeometryRecord> m_DD4Hep;
mutable edm::ESGetToken<HepPDT::ParticleDataTable, PDTRecord> m_PDT;

// status flags
mutable std::mutex m_protectMutex;
mutable std::mutex m_threadMutex;
mutable std::condition_variable m_notifyMasterCv;
mutable std::condition_variable m_notifyMainCv;

mutable ThreadState m_masterThreadState;

mutable bool m_masterCanProceed;
mutable bool m_mainCanProceed;
mutable bool m_firstRun;
mutable bool m_stopped;
mutable bool m_hasToken = false;
mutable bool m_masterCanProceed = false;
mutable bool m_mainCanProceed = false;
mutable bool m_firstRun = true;
mutable bool m_stopped = false;
};

#endif
4 changes: 0 additions & 4 deletions SimG4Core/Application/interface/RunManagerMT.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@
#define SimG4Core_RunManagerMT_H

#include "FWCore/Framework/interface/Event.h"
#include "DataFormats/Common/interface/Handle.h"
#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/Utilities/interface/InputTag.h"

#include "SimG4Core/Geometry/interface/SensitiveDetectorCatalog.h"

Expand Down Expand Up @@ -59,7 +56,6 @@ class RunManagerMT {
explicit RunManagerMT(edm::ParameterSet const&);
~RunManagerMT();

// void initG4(const DDCompactView*, const cms::DDCompactView*, const MagneticField*, const HepPDT::ParticleDataTable*);
void initG4(const DDCompactView*, const cms::DDCompactView*, const HepPDT::ParticleDataTable*);

void initializeUserActions();
Expand Down
8 changes: 7 additions & 1 deletion SimG4Core/Application/interface/RunManagerMTWorker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#include "SimG4Core/Generators/interface/Generator.h"
#include "SimDataFormats/Forward/interface/LHCTransportLinkContainer.h"

#include "MagneticField/Engine/interface/MagneticField.h"
#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"

#include <memory>
#include <tbb/concurrent_vector.h>
#include <unordered_map>
Expand All @@ -19,6 +22,7 @@ namespace edm {
class ConsumesCollector;
class HepMCProduct;
} // namespace edm

class Generator;
class RunManagerMT;

Expand All @@ -43,7 +47,7 @@ class SimProducer;

class RunManagerMTWorker {
public:
explicit RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::ConsumesCollector&& i);
explicit RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::ConsumesCollector&& iC);
~RunManagerMTWorker();

void beginRun(const edm::EventSetup&);
Expand Down Expand Up @@ -88,6 +92,8 @@ class RunManagerMTWorker {
edm::EDGetTokenT<edm::HepMCProduct> m_InToken;
edm::EDGetTokenT<edm::HepMCProduct> m_LHCToken;
edm::EDGetTokenT<edm::LHCTransportLinkContainer> m_theLHCTlinkToken;
edm::ESGetToken<MagneticField, IdealMagneticFieldRecord> m_MagField;
const MagneticField* m_pMagField = nullptr;

bool m_nonBeam;
bool m_pUseMagneticField;
Expand Down
37 changes: 17 additions & 20 deletions SimG4Core/Application/plugins/OscarMTProducer.cc
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,30 @@
#include "FWCore/PluginManager/interface/PluginManager.h"
#include "OscarMTProducer.h"

#include "FWCore/PluginManager/interface/PluginManager.h"
#include "FWCore/Framework/interface/MakerMacros.h"
#include "FWCore/Framework/interface/ConsumesCollector.h"
#include "FWCore/ParameterSet/interface/ParameterSet.h"

#include "OscarMTProducer.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"
#include "FWCore/ServiceRegistry/interface/Service.h"
#include "FWCore/Utilities/interface/RandomNumberGenerator.h"
#include "FWCore/Utilities/interface/Exception.h"

#include "SimG4Core/Notification/interface/SimG4Exception.h"
#include "SimG4Core/Application/interface/RunManagerMT.h"
#include "SimG4Core/Application/interface/RunManagerMTWorker.h"
#include "SimG4Core/Notification/interface/G4SimEvent.h"
#include "SimG4Core/SensitiveDetector/interface/SensitiveTkDetector.h"
#include "SimG4Core/SensitiveDetector/interface/SensitiveCaloDetector.h"
#include "SimG4Core/Watcher/interface/SimProducer.h"

#include "SimDataFormats/Track/interface/SimTrackContainer.h"
#include "SimDataFormats/Vertex/interface/SimVertexContainer.h"
#include "SimDataFormats/TrackingHit/interface/PSimHitContainer.h"
#include "SimDataFormats/CaloHit/interface/PCaloHitContainer.h"

#include "SimG4Core/Watcher/interface/SimProducer.h"

#include "FWCore/Utilities/interface/Exception.h"
#include "SimG4Core/Notification/interface/SimG4Exception.h"

#include "FWCore/ServiceRegistry/interface/Service.h"
#include "FWCore/Utilities/interface/RandomNumberGenerator.h"
#include "Randomize.hh"

#include "FWCore/MessageLogger/interface/MessageLogger.h"

#include <iostream>
#include <memory>

Expand Down Expand Up @@ -70,7 +67,9 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster

m_runManagerWorker = std::make_unique<RunManagerMTWorker>(p, consumesCollector());
m_masterThread = ms;
m_masterThread->callConsumes(consumesCollector());

// List of produced containers
produces<edm::SimTrackContainer>().setBranchAlias("SimTracks");
produces<edm::SimVertexContainer>().setBranchAlias("SimVertices");
produces<edm::PSimHitContainer>("TrackerHitsPixelBarrelLowTof");
Expand Down Expand Up @@ -128,9 +127,8 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster

//register any products
auto& producers = m_runManagerWorker->producers();

for (Producers::iterator itProd = producers.begin(); itProd != producers.end(); ++itProd) {
(*itProd)->registerProducts(producesCollector());
for (auto& ptr : producers) {
ptr->registerProducts(producesCollector());
}
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer is constructed";
}
Expand All @@ -145,19 +143,18 @@ std::unique_ptr<OscarMTMasterThread> OscarMTProducer::initializeGlobalCache(cons
return std::make_unique<OscarMTMasterThread>(iConfig);
}

std::shared_ptr<int> OscarMTProducer::globalBeginRun(const edm::Run& iRun,
std::shared_ptr<int> OscarMTProducer::globalBeginRun(const edm::Run&,
const edm::EventSetup& iSetup,
const OscarMTMasterThread* masterThread) {
// Random number generation not allowed here
StaticRandomEngineSetUnset random(nullptr);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun";
masterThread->beginRun(iSetup);
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun done";
return std::shared_ptr<int>();
}

void OscarMTProducer::globalEndRun(const edm::Run& iRun, const edm::EventSetup& iSetup, const RunContext* iContext) {
void OscarMTProducer::globalEndRun(const edm::Run&, const edm::EventSetup&, const RunContext* iContext) {
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalEndRun";
iContext->global()->endRun();
}
Expand Down Expand Up @@ -185,7 +182,7 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) {
void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
StaticRandomEngineSetUnset random(e.streamID());
edm::LogVerbatim("SimG4CoreApplication") << "Produce event " << e.id() << " stream " << e.streamID();
LogDebug("SimG4CoreApplication") << "Before event rand= " << G4UniformRand();
//edm::LogVerbatim("SimG4CoreApplication") << " rand= " << G4UniformRand();

auto& sTk = m_runManagerWorker->sensTkDetectors();
auto& sCalo = m_runManagerWorker->sensCaloDetectors();
Expand Down Expand Up @@ -215,7 +212,7 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
std::unique_ptr<edm::PSimHitContainer> product(new edm::PSimHitContainer);
tracker->fillHits(*product, name);
if (product != nullptr && !product->empty())
edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " traker hits <" << name << ">";
edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " tracker hits <" << name << ">";
e.put(std::move(product), name);
}
}
Expand All @@ -236,7 +233,7 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
prod.get()->produce(e, es);
}
edm::LogVerbatim("SimG4CoreApplication") << "Event is produced " << e.id() << " stream " << e.streamID();
LogDebug("SimG4CoreApplication") << "End of event rand= " << G4UniformRand();
//edm::LogVerbatim("SimG4CoreApplication") << " rand= " << G4UniformRand();
}

StaticRandomEngineSetUnset::StaticRandomEngineSetUnset(edm::StreamID const& streamID) {
Expand Down
1 change: 0 additions & 1 deletion SimG4Core/Application/plugins/OscarMTProducer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

#include "FWCore/Framework/interface/stream/EDProducer.h"
#include "FWCore/Framework/interface/Event.h"
#include "DataFormats/Common/interface/Handle.h"
#include "FWCore/Framework/interface/MakerMacros.h"
#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/Framework/interface/Run.h"
Expand Down
90 changes: 32 additions & 58 deletions SimG4Core/Application/src/OscarMTMasterThread.cc
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,24 @@
#include <memory>

#include "SimG4Core/Application/interface/OscarMTMasterThread.h"
#include "FWCore/Utilities/interface/EDMException.h"
#include "FWCore/Framework/interface/ConsumesCollector.h"

#include "SimG4Core/Application/interface/RunManagerMT.h"
#include "SimG4Core/Geometry/interface/CustomUIsession.h"

#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/Framework/interface/ESHandle.h"
#include "FWCore/Framework/interface/ESTransientHandle.h"
#include "FWCore/Utilities/interface/EDMException.h"

#include "Geometry/Records/interface/IdealGeometryRecord.h"
#include "DetectorDescription/Core/interface/DDCompactView.h"
#include "DetectorDescription/DDCMS/interface/DDCompactView.h"

#include "HepPDT/ParticleDataTable.hh"
#include "SimGeneral/HepPDTRecord/interface/PDTRecord.h"

#include "G4PhysicalVolumeStore.hh"

OscarMTMasterThread::OscarMTMasterThread(const edm::ParameterSet& iConfig)
: m_pGeoFromDD4hep(iConfig.getParameter<bool>("g4GeometryDD4hepSource")),
m_pDD(nullptr),
m_pDD4hep(nullptr),
m_pTable(nullptr),
m_masterThreadState(ThreadState::NotExist),
m_masterCanProceed(false),
m_mainCanProceed(false),
m_firstRun(true),
m_stopped(false) {
m_masterThreadState(ThreadState::NotExist) {
// Lock the mutex
std::unique_lock<std::mutex> lk(m_threadMutex);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread: creating master thread";
edm::LogVerbatim("SimG4CoreApplication")
<< "OscarMTMasterThread: creating master thread DD4Hep: " << m_pGeoFromDD4hep;

// Create Genat4 master thread
// Create Geant4 master thread
m_masterThread = std::thread([&]() {
/////////////////
// Initialization
Expand Down Expand Up @@ -64,14 +48,15 @@ OscarMTMasterThread::OscarMTMasterThread(const edm::ParameterSet& iConfig)
m_masterCanProceed = false;
edm::LogVerbatim("OscarMTMasterThread") << "Master thread: State loop, starting wait";
m_notifyMasterCv.wait(lk2, [&] { return m_masterCanProceed; });
//m_notifyMasterCv.wait(lk2, [&] { return false; });

// Act according to the state
edm::LogVerbatim("OscarMTMasterThread")
<< "Master thread: Woke up, state is " << static_cast<int>(m_masterThreadState);
if (m_masterThreadState == ThreadState::BeginRun) {
// Initialize Geant4
edm::LogVerbatim("OscarMTMasterThread") << "Master thread: Initializing Geant4";
m_runManagerMaster->initG4(m_pDD, m_pDD4hep, m_pTable);
m_runManagerMaster->initG4(m_pDDD, m_pDD4Hep, m_pTable);
isG4Alive = true;
} else if (m_masterThreadState == ThreadState::EndRun) {
// Stop Geant4
Expand Down Expand Up @@ -119,14 +104,33 @@ OscarMTMasterThread::~OscarMTMasterThread() {
}
}

void OscarMTMasterThread::callConsumes(edm::ConsumesCollector&& iC) const {
if (m_hasToken) {
return;
}
if (m_pGeoFromDD4hep) {
m_DD4Hep = iC.esConsumes<cms::DDCompactView, IdealGeometryRecord, edm::Transition::BeginRun>();
} else {
m_DDD = iC.esConsumes<DDCompactView, IdealGeometryRecord, edm::Transition::BeginRun>();
}
m_PDT = iC.esConsumes<HepPDT::ParticleDataTable, PDTRecord, edm::Transition::BeginRun>();
m_hasToken = true;
}

void OscarMTMasterThread::beginRun(const edm::EventSetup& iSetup) const {
std::lock_guard<std::mutex> lk(m_protectMutex);

std::unique_lock<std::mutex> lk2(m_threadMutex);
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread::beginRun";

// Reading from ES must be done in the main (CMSSW) thread
readES(iSetup);

if (m_firstRun) {
if (m_pGeoFromDD4hep) {
m_pDD4Hep = &(*iSetup.getTransientHandle(m_DD4Hep));
} else {
m_pDDD = &(*iSetup.getTransientHandle(m_DDD));
}
m_pTable = &iSetup.getData(m_PDT);
m_firstRun = false;
}
m_masterThreadState = ThreadState::BeginRun;
m_masterCanProceed = true;
m_mainCanProceed = false;
Expand All @@ -140,8 +144,8 @@ void OscarMTMasterThread::beginRun(const edm::EventSetup& iSetup) const {

void OscarMTMasterThread::endRun() const {
std::lock_guard<std::mutex> lk(m_protectMutex);

std::unique_lock<std::mutex> lk2(m_threadMutex);

m_masterThreadState = ThreadState::EndRun;
m_mainCanProceed = false;
m_masterCanProceed = true;
Expand Down Expand Up @@ -176,33 +180,3 @@ void OscarMTMasterThread::stopThread() {
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread::stopTread: main thread finished";
m_stopped = true;
}

void OscarMTMasterThread::readES(const edm::EventSetup& iSetup) const {
bool geomChanged = idealGeomRcdWatcher_.check(iSetup);
if (geomChanged && (!m_firstRun)) {
throw edm::Exception(edm::errors::Configuration)
<< "[SimG4Core OscarMTMasterThread]\n"
<< "The Geometry configuration is changed during the job execution\n"
<< "this is not allowed, the geometry must stay unchanged";
}
// Don't read from ES if not the first run, just as in
if (!m_firstRun)
return;

// DDDWorld: get the DDCV from the ES and use it to build the World
if (m_pGeoFromDD4hep) {
edm::ESTransientHandle<cms::DDCompactView> pDD;
iSetup.get<IdealGeometryRecord>().get(pDD);
m_pDD4hep = pDD.product();
} else {
edm::ESTransientHandle<DDCompactView> pDD;
iSetup.get<IdealGeometryRecord>().get(pDD);
m_pDD = pDD.product();
}

edm::ESHandle<HepPDT::ParticleDataTable> fTable;
iSetup.get<PDTRecord>().get(fTable);
m_pTable = fTable.product();

m_firstRun = false;
}
Loading

0 comments on commit 639903d

Please sign in to comment.