Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use of ESGetToken in Oscar producer #34338

Merged
merged 16 commits into from
Jul 13, 2021
Merged

Use of ESGetToken in Oscar producer #34338

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
87a6606
migrate event setup
civanch Jul 5, 2021
ccc45f8
code checks and remove unused headers
civanch Jul 5, 2021
03f4773
added forgotten
civanch Jul 5, 2021
3bf6f9a
implemented suggestings
civanch Jul 6, 2021
dd9c00b
code-checks
civanch Jul 6, 2021
97aa342
code-checks
civanch Jul 6, 2021
7c73629
provide regression
civanch Jul 6, 2021
c150a5f
code-checks
civanch Jul 6, 2021
44fe543
removed last static member from OscarMTProducer
civanch Jul 6, 2021
deffd8c
tokens moved to OscarMTMasterThread
civanch Jul 6, 2021
3706a26
clean-up
civanch Jul 8, 2021
0ddcfd3
updated
civanch Jul 8, 2021
b7aadfb
moved all consumes to the master thread
civanch Jul 9, 2021
ce5dfbf
code-checks
civanch Jul 9, 2021
97e773a
return back mmagnetic field to worker
civanch Jul 12, 2021
3ee5f11
fixed compilation
civanch Jul 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions SimG4Core/Application/interface/OscarMTMasterThread.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/Framework/interface/ESWatcher.h"

#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"
#include "Geometry/Records/interface/IdealGeometryRecord.h"
#include "DetectorDescription/Core/interface/DDCompactView.h"
#include "DetectorDescription/DDCMS/interface/DDCompactView.h"

#include "HepPDT/ParticleDataTable.hh"
#include "SimGeneral/HepPDTRecord/interface/PDTRecord.h"

#include <memory>
#include <thread>
Expand Down Expand Up @@ -34,16 +37,15 @@ class OscarMTMasterThread {
explicit OscarMTMasterThread(const edm::ParameterSet& iConfig);
~OscarMTMasterThread();

void beginRun(const edm::EventSetup& iSetup) const;
void beginRun(const DDCompactView*, const cms::DDCompactView*, const HepPDT::ParticleDataTable*) const;
void endRun() const;
void stopThread();

inline RunManagerMT& runManagerMaster() const { return *m_runManagerMaster; }
inline RunManagerMT* runManagerMasterPtr() const { return m_runManagerMaster.get(); }
inline bool isDD4Hep() const { return m_pGeoFromDD4hep; }

private:
void readES(const edm::EventSetup& iSetup) const;

enum class ThreadState { NotExist = 0, BeginRun = 1, EndRun = 2, Destruct = 3 };

const bool m_pGeoFromDD4hep;
Expand All @@ -52,23 +54,22 @@ class OscarMTMasterThread {
std::thread m_masterThread;

// ES products needed for Geant4 initialization
mutable edm::ESWatcher<IdealGeometryRecord> idealGeomRcdWatcher_;
mutable edm::ESWatcher<IdealMagneticFieldRecord> idealMagRcdWatcher_;
mutable const DDCompactView* m_pDD;
mutable const cms::DDCompactView* m_pDD4hep;
mutable const HepPDT::ParticleDataTable* m_pTable;
mutable const DDCompactView* m_pDDD = nullptr;
mutable const cms::DDCompactView* m_pDD4Hep = nullptr;
mutable const HepPDT::ParticleDataTable* m_pTable = nullptr;

// status flags
mutable std::mutex m_protectMutex;
mutable std::mutex m_threadMutex;
mutable std::condition_variable m_notifyMasterCv;
mutable std::condition_variable m_notifyMainCv;

mutable ThreadState m_masterThreadState;

mutable bool m_masterCanProceed;
mutable bool m_mainCanProceed;
mutable bool m_firstRun;
mutable bool m_stopped;
mutable bool m_masterCanProceed = false;
mutable bool m_mainCanProceed = false;
mutable bool m_firstRun = true;
civanch marked this conversation as resolved.
Show resolved Hide resolved
mutable bool m_stopped = false;
};

#endif
4 changes: 0 additions & 4 deletions SimG4Core/Application/interface/RunManagerMT.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@
#define SimG4Core_RunManagerMT_H

#include "FWCore/Framework/interface/Event.h"
#include "DataFormats/Common/interface/Handle.h"
#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/ParameterSet/interface/ParameterSet.h"
#include "FWCore/Utilities/interface/InputTag.h"

#include "SimG4Core/Geometry/interface/SensitiveDetectorCatalog.h"

Expand Down Expand Up @@ -59,7 +56,6 @@ class RunManagerMT {
explicit RunManagerMT(edm::ParameterSet const&);
~RunManagerMT();

// void initG4(const DDCompactView*, const cms::DDCompactView*, const MagneticField*, const HepPDT::ParticleDataTable*);
void initG4(const DDCompactView*, const cms::DDCompactView*, const HepPDT::ParticleDataTable*);

void initializeUserActions();
Expand Down
7 changes: 7 additions & 0 deletions SimG4Core/Application/interface/RunManagerMTWorker.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#include "SimG4Core/Generators/interface/Generator.h"
#include "SimDataFormats/Forward/interface/LHCTransportLinkContainer.h"

#include "MagneticField/Engine/interface/MagneticField.h"
#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"

#include <memory>
#include <tbb/concurrent_vector.h>
#include <unordered_map>
Expand All @@ -19,6 +22,7 @@ namespace edm {
class ConsumesCollector;
class HepMCProduct;
} // namespace edm

class Generator;
class RunManagerMT;

Expand Down Expand Up @@ -70,6 +74,8 @@ class RunManagerMTWorker {

void initializeG4(RunManagerMT* runManagerMaster, const edm::EventSetup& es);

inline void SetMagFieldToken(edm::ESGetToken<MagneticField, IdealMagneticFieldRecord>& ref) { m_MagField = ref; }

private:
void initializeTLS();
void initializeUserActions();
Expand All @@ -88,6 +94,7 @@ class RunManagerMTWorker {
edm::EDGetTokenT<edm::HepMCProduct> m_InToken;
edm::EDGetTokenT<edm::HepMCProduct> m_LHCToken;
edm::EDGetTokenT<edm::LHCTransportLinkContainer> m_theLHCTlinkToken;
edm::ESGetToken<MagneticField, IdealMagneticFieldRecord> m_MagField;

bool m_nonBeam;
bool m_pUseMagneticField;
Expand Down
60 changes: 47 additions & 13 deletions SimG4Core/Application/plugins/OscarMTProducer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,13 @@
#include <iostream>
#include <memory>

edm::ESGetToken<cms::DDCompactView, IdealGeometryRecord> OscarMTProducer::m_DD4Hep;
edm::ESGetToken<DDCompactView, IdealGeometryRecord> OscarMTProducer::m_DDD;
edm::ESGetToken<HepPDT::ParticleDataTable, PDTRecord> OscarMTProducer::m_PDT;
edm::ESGetToken<MagneticField, IdealMagneticFieldRecord> OscarMTProducer::m_MagField;
G4Mutex OscarMTProducer::m_OscarMutex = G4MUTEX_INITIALIZER;
bool OscarMTProducer::m_hasToken = false;
civanch marked this conversation as resolved.
Show resolved Hide resolved

namespace edm {
class StreamID;
}
Expand Down Expand Up @@ -71,6 +78,24 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster
m_runManagerWorker = std::make_unique<RunManagerMTWorker>(p, consumesCollector());
m_masterThread = ms;

// Prepare tokens
bool isDD4Hep = p.getParameter<bool>("g4GeometryDD4hepSource");
if (!m_hasToken) {
G4MUTEXLOCK(&m_OscarMutex);
civanch marked this conversation as resolved.
Show resolved Hide resolved
if (!m_hasToken) {
if (isDD4Hep) {
m_DD4Hep = esConsumes<cms::DDCompactView, IdealGeometryRecord, edm::Transition::BeginRun>();
} else {
m_DDD = esConsumes<DDCompactView, IdealGeometryRecord, edm::Transition::BeginRun>();
}
m_PDT = esConsumes<HepPDT::ParticleDataTable, PDTRecord, edm::Transition::BeginRun>();
m_MagField = esConsumes<MagneticField, IdealMagneticFieldRecord, edm::Transition::BeginRun>();
}
G4MUTEXUNLOCK(&m_OscarMutex);
}
m_runManagerWorker->SetMagFieldToken(m_MagField);

// List of produced containers
produces<edm::SimTrackContainer>().setBranchAlias("SimTracks");
produces<edm::SimVertexContainer>().setBranchAlias("SimVertices");
produces<edm::PSimHitContainer>("TrackerHitsPixelBarrelLowTof");
Expand Down Expand Up @@ -128,11 +153,10 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster

//register any products
auto& producers = m_runManagerWorker->producers();

for (Producers::iterator itProd = producers.begin(); itProd != producers.end(); ++itProd) {
(*itProd)->registerProducts(producesCollector());
for (auto& ptr : producers) {
ptr->registerProducts(producesCollector());
}
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer is constructed";
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer is constructed DD4Hep: " << isDD4Hep;
}

OscarMTProducer::~OscarMTProducer() {}
Expand All @@ -145,19 +169,29 @@ std::unique_ptr<OscarMTMasterThread> OscarMTProducer::initializeGlobalCache(cons
return std::make_unique<OscarMTMasterThread>(iConfig);
}

std::shared_ptr<int> OscarMTProducer::globalBeginRun(const edm::Run& iRun,
std::shared_ptr<int> OscarMTProducer::globalBeginRun(const edm::Run&,
const edm::EventSetup& iSetup,
const OscarMTMasterThread* masterThread) {
// Random number generation not allowed here
StaticRandomEngineSetUnset random(nullptr);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun";
masterThread->beginRun(iSetup);

// Prepare master thread
const DDCompactView* pDDD = nullptr;
const cms::DDCompactView* pDD4Hep = nullptr;
if (masterThread->isDD4Hep()) {
pDD4Hep = &(iSetup.getData(m_DD4Hep));
} else {
pDDD = &(iSetup.getData(m_DDD));
}
const HepPDT::ParticleDataTable* pPDT = &(iSetup.getData(m_PDT));
masterThread->beginRun(pDDD, pDD4Hep, pPDT);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalBeginRun done";
return std::shared_ptr<int>();
}

void OscarMTProducer::globalEndRun(const edm::Run& iRun, const edm::EventSetup& iSetup, const RunContext* iContext) {
void OscarMTProducer::globalEndRun(const edm::Run&, const edm::EventSetup&, const RunContext* iContext) {
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::globalEndRun";
iContext->global()->endRun();
}
Expand All @@ -184,8 +218,8 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) {

void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
StaticRandomEngineSetUnset random(e.streamID());
edm::LogVerbatim("SimG4CoreApplication") << "Produce event " << e.id() << " stream " << e.streamID();
LogDebug("SimG4CoreApplication") << "Before event rand= " << G4UniformRand();
edm::LogVerbatim("SimG4CoreApplication")
<< "Produce event " << e.id() << " stream " << e.streamID() << " rand= " << G4UniformRand();

auto& sTk = m_runManagerWorker->sensTkDetectors();
auto& sCalo = m_runManagerWorker->sensCaloDetectors();
Expand Down Expand Up @@ -215,7 +249,7 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
std::unique_ptr<edm::PSimHitContainer> product(new edm::PSimHitContainer);
tracker->fillHits(*product, name);
if (product != nullptr && !product->empty())
edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " traker hits <" << name << ">";
edm::LogVerbatim("SimG4CoreApplication") << "Produced " << product->size() << " tracker hits <" << name << ">";
e.put(std::move(product), name);
}
}
Expand All @@ -235,8 +269,8 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) {
for (auto& prod : producers) {
prod.get()->produce(e, es);
}
edm::LogVerbatim("SimG4CoreApplication") << "Event is produced " << e.id() << " stream " << e.streamID();
LogDebug("SimG4CoreApplication") << "End of event rand= " << G4UniformRand();
edm::LogVerbatim("SimG4CoreApplication")
<< "Event is produced " << e.id() << " stream " << e.streamID() << " rand= " << G4UniformRand();
}

StaticRandomEngineSetUnset::StaticRandomEngineSetUnset(edm::StreamID const& streamID) {
Expand Down
20 changes: 19 additions & 1 deletion SimG4Core/Application/plugins/OscarMTProducer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,24 @@

#include "FWCore/Framework/interface/stream/EDProducer.h"
#include "FWCore/Framework/interface/Event.h"
#include "DataFormats/Common/interface/Handle.h"
#include "FWCore/Framework/interface/MakerMacros.h"
#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/Framework/interface/Run.h"

#include "SimG4Core/Application/interface/OscarMTMasterThread.h"

#include "DetectorDescription/Core/interface/DDCompactView.h"
#include "DetectorDescription/DDCMS/interface/DDCompactView.h"
#include "Geometry/Records/interface/IdealGeometryRecord.h"

#include "HepPDT/ParticleDataTable.hh"
#include "SimGeneral/HepPDTRecord/interface/PDTRecord.h"

#include "MagneticField/Engine/interface/MagneticField.h"
#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h"

#include "G4Threading.hh"

#include <memory>

class SimProducer;
Expand Down Expand Up @@ -38,6 +49,13 @@ class OscarMTProducer : public edm::stream::EDProducer<edm::GlobalCache<OscarMTM
private:
std::unique_ptr<RunManagerMTWorker> m_runManagerWorker;
const OscarMTMasterThread* m_masterThread = nullptr;

static edm::ESGetToken<cms::DDCompactView, IdealGeometryRecord> m_DD4Hep;
civanch marked this conversation as resolved.
Show resolved Hide resolved
static edm::ESGetToken<DDCompactView, IdealGeometryRecord> m_DDD;
static edm::ESGetToken<HepPDT::ParticleDataTable, PDTRecord> m_PDT;
static edm::ESGetToken<MagneticField, IdealMagneticFieldRecord> m_MagField;
static G4Mutex m_OscarMutex;
static bool m_hasToken;
};

#endif
65 changes: 11 additions & 54 deletions SimG4Core/Application/src/OscarMTMasterThread.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,19 @@
#include "SimG4Core/Application/interface/RunManagerMT.h"
#include "SimG4Core/Geometry/interface/CustomUIsession.h"

#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/Framework/interface/ESHandle.h"
#include "FWCore/Framework/interface/ESTransientHandle.h"
#include "FWCore/Utilities/interface/EDMException.h"

#include "Geometry/Records/interface/IdealGeometryRecord.h"
#include "DetectorDescription/Core/interface/DDCompactView.h"
#include "DetectorDescription/DDCMS/interface/DDCompactView.h"

#include "HepPDT/ParticleDataTable.hh"
#include "SimGeneral/HepPDTRecord/interface/PDTRecord.h"

#include "G4PhysicalVolumeStore.hh"

OscarMTMasterThread::OscarMTMasterThread(const edm::ParameterSet& iConfig)
: m_pGeoFromDD4hep(iConfig.getParameter<bool>("g4GeometryDD4hepSource")),
m_pDD(nullptr),
m_pDD4hep(nullptr),
m_pTable(nullptr),
m_masterThreadState(ThreadState::NotExist),
m_masterCanProceed(false),
m_mainCanProceed(false),
m_firstRun(true),
m_stopped(false) {
m_masterThreadState(ThreadState::NotExist) {
// Lock the mutex
std::unique_lock<std::mutex> lk(m_threadMutex);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread: creating master thread";

// Create Genat4 master thread
// Create Geant4 master thread
m_masterThread = std::thread([&]() {
/////////////////
// Initialization
Expand Down Expand Up @@ -71,7 +54,7 @@ OscarMTMasterThread::OscarMTMasterThread(const edm::ParameterSet& iConfig)
if (m_masterThreadState == ThreadState::BeginRun) {
// Initialize Geant4
edm::LogVerbatim("OscarMTMasterThread") << "Master thread: Initializing Geant4";
m_runManagerMaster->initG4(m_pDD, m_pDD4hep, m_pTable);
m_runManagerMaster->initG4(m_pDDD, m_pDD4Hep, m_pTable);
isG4Alive = true;
} else if (m_masterThreadState == ThreadState::EndRun) {
// Stop Geant4
Expand Down Expand Up @@ -119,17 +102,21 @@ OscarMTMasterThread::~OscarMTMasterThread() {
}
}

void OscarMTMasterThread::beginRun(const edm::EventSetup& iSetup) const {
void OscarMTMasterThread::beginRun(const DDCompactView* pDDD,
const cms::DDCompactView* pDD4Hep,
const HepPDT::ParticleDataTable* pPDT) const {
std::lock_guard<std::mutex> lk(m_protectMutex);

std::unique_lock<std::mutex> lk2(m_threadMutex);

// Reading from ES must be done in the main (CMSSW) thread
readES(iSetup);

edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread::beginRun";
m_pDDD = pDDD;
m_pDD4Hep = pDD4Hep;
m_pTable = pPDT;
m_masterThreadState = ThreadState::BeginRun;
m_masterCanProceed = true;
m_mainCanProceed = false;
m_firstRun = false;
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread: Signal master for BeginRun";
m_notifyMasterCv.notify_one();
m_notifyMainCv.wait(lk2, [&]() { return m_mainCanProceed; });
Expand Down Expand Up @@ -176,33 +163,3 @@ void OscarMTMasterThread::stopThread() {
edm::LogVerbatim("SimG4CoreApplication") << "OscarMTMasterThread::stopTread: main thread finished";
m_stopped = true;
}

void OscarMTMasterThread::readES(const edm::EventSetup& iSetup) const {
bool geomChanged = idealGeomRcdWatcher_.check(iSetup);
if (geomChanged && (!m_firstRun)) {
throw edm::Exception(edm::errors::Configuration)
<< "[SimG4Core OscarMTMasterThread]\n"
<< "The Geometry configuration is changed during the job execution\n"
<< "this is not allowed, the geometry must stay unchanged";
}
// Don't read from ES if not the first run, just as in
if (!m_firstRun)
return;

// DDDWorld: get the DDCV from the ES and use it to build the World
if (m_pGeoFromDD4hep) {
edm::ESTransientHandle<cms::DDCompactView> pDD;
iSetup.get<IdealGeometryRecord>().get(pDD);
m_pDD4hep = pDD.product();
} else {
edm::ESTransientHandle<DDCompactView> pDD;
iSetup.get<IdealGeometryRecord>().get(pDD);
m_pDD = pDD.product();
}

edm::ESHandle<HepPDT::ParticleDataTable> fTable;
iSetup.get<PDTRecord>().get(fTable);
m_pTable = fTable.product();

m_firstRun = false;
}
Loading