Skip to content

Commit

Permalink
Merge pull request #36229 from civanch/sim_watcher_update
Browse files Browse the repository at this point in the history 
SimWatcher thread safe interface
  • Loading branch information
@cmsbuild
cmsbuild authored Nov 25, 2021
2 parents 6d262f0 + ed34236 commit 5a0d088
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 52 deletions.
22 changes: 11 additions & 11 deletions SimG4Core/Application/interface/RunManagerMTWorker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,20 +87,23 @@ class RunManagerMTWorker {
void DumpMagneticField(const G4Field*, const std::string&) const;

void resetTLS();
int getThreadIndex() const { return m_thread_index; }
inline int getThreadIndex() const { return m_thread_index; }

Generator m_generator;
edm::EDGetTokenT<edm::HepMCProduct> m_InToken;
edm::EDGetTokenT<edm::HepMCProduct> m_LHCToken;
edm::EDGetTokenT<edm::LHCTransportLinkContainer> m_theLHCTlinkToken;
edm::ESGetToken<MagneticField, IdealMagneticFieldRecord> m_MagField;
const MagneticField* m_pMagField = nullptr;
const MagneticField* m_pMagField{nullptr};

bool m_nonBeam;
bool m_pUseMagneticField;
bool m_hasWatchers;
bool m_LHCTransport;
int m_EvtMgrVerbosity;
bool m_nonBeam{false};
bool m_pUseMagneticField{true};
bool m_hasWatchers{false};
bool m_LHCTransport{false};
bool m_dumpMF{false};

const int m_thread_index{-1};
int m_EvtMgrVerbosity{0};

edm::ParameterSet m_pField;
edm::ParameterSet m_pRunAction;
Expand All @@ -113,13 +116,10 @@ class RunManagerMTWorker {

struct TLSData;
TLSData* m_tls{nullptr};
bool dumpMF{false};

G4SimEvent* m_simEvent;
G4SimEvent* m_simEvent{nullptr};
std::unique_ptr<CMSSteppingVerbose> m_sVerbose;
std::unordered_map<std::string, std::unique_ptr<SensitiveDetectorMakerBase>> m_sdMakers;

const int m_thread_index{-1};
};

#endif
2 changes: 1 addition & 1 deletion SimG4Core/Application/src/RunManagerMT.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ void RunManagerMT::initG4(const DDCompactView* pDD,
}
m_kernel->DefineWorldVolume(world, true);
m_registry.dddWorldSignal_(m_world.get());
G4StateManager::GetStateManager()->SetNewState(G4State_PreInit);
m_stateManager->SetNewState(G4State_PreInit);

// Create physics list
edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMT: create PhysicsList";
Expand Down
104 changes: 64 additions & 40 deletions SimG4Core/Application/src/RunManagerMTWorker.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,31 +71,42 @@ namespace {

int get_new_thread_index() { return thread_counter++; }

void createWatchers(const edm::ParameterSet& iP,
bool createWatchers(const edm::ParameterSet& iP,
SimActivityRegistry* iReg,
std::vector<std::shared_ptr<SimWatcher>>& oWatchers,
std::vector<std::shared_ptr<SimProducer>>& oProds) {
if (!iP.exists("Watchers")) {
return;
std::vector<std::shared_ptr<SimProducer>>& oProds,
int threadID) {
std::vector<edm::ParameterSet> ws = iP.getParameter<std::vector<edm::ParameterSet>>("Watchers");
if (ws.empty()) {
return false;
}

std::vector<edm::ParameterSet> watchers = iP.getParameter<std::vector<edm::ParameterSet>>("Watchers");

for (auto& watcher : watchers) {
for (auto& watcher : ws) {
std::unique_ptr<SimWatcherMakerBase> maker(
SimWatcherFactory::get()->create(watcher.getParameter<std::string>("type")));
if (maker == nullptr) {
throw edm::Exception(edm::errors::Configuration)
<< "Unable to find the requested Watcher <" << watcher.getParameter<std::string>("type");
}
std::shared_ptr<SimWatcher> watcherTemp;
std::shared_ptr<SimProducer> producerTemp;
maker->make(watcher, *(iReg), watcherTemp, producerTemp);
oWatchers.push_back(watcherTemp);
if (producerTemp) {
oProds.push_back(producerTemp);
<< "RunManagerMTWorker::createWatchers: "
<< "Unable to find the requested Watcher " << watcher.getParameter<std::string>("type");
} else {
std::shared_ptr<SimWatcher> watcherTemp;
std::shared_ptr<SimProducer> producerTemp;
maker->make(watcher, *(iReg), watcherTemp, producerTemp);
if (nullptr != watcherTemp) {
if (!watcherTemp->isMT() && 0 < threadID) {
throw edm::Exception(edm::errors::Configuration)
<< "RunManagerMTWorker::createWatchers: "
<< "Unable to use Watcher " << watcher.getParameter<std::string>("type") << " if number of threads > 1";
} else {
oWatchers.push_back(watcherTemp);
if (nullptr != producerTemp) {
oProds.push_back(producerTemp);
}
}
}
}
}
return (!oWatchers.empty());
}
} // namespace

Expand Down Expand Up @@ -139,6 +150,7 @@ RunManagerMTWorker::RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::Co
m_nonBeam(iConfig.getParameter<bool>("NonBeamEvent")),
m_pUseMagneticField(iConfig.getParameter<bool>("UseMagneticField")),
m_LHCTransport(iConfig.getParameter<bool>("LHCTransport")),
m_thread_index{get_new_thread_index()},
m_EvtMgrVerbosity(iConfig.getUntrackedParameter<int>("G4EventManagerVerbosity", 0)),
m_pField(iConfig.getParameter<edm::ParameterSet>("MagneticField")),
m_pRunAction(iConfig.getParameter<edm::ParameterSet>("RunAction")),
Expand All @@ -147,16 +159,22 @@ RunManagerMTWorker::RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::Co
m_pTrackingAction(iConfig.getParameter<edm::ParameterSet>("TrackingAction")),
m_pSteppingAction(iConfig.getParameter<edm::ParameterSet>("SteppingAction")),
m_pCustomUIsession(iConfig.getUntrackedParameter<edm::ParameterSet>("CustomUIsession")),
m_p(iConfig),
m_simEvent(nullptr),
m_sVerbose(nullptr),
m_thread_index{get_new_thread_index()} {
m_p(iConfig) {
int thisID = getThreadIndex();
edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMTWorker for the thread " << thisID;

// sensitive detectors
std::vector<std::string> onlySDs = iConfig.getParameter<std::vector<std::string>>("OnlySDs");
m_sdMakers = sim::sensitiveDetectorMakers(m_p, iC, onlySDs);
std::vector<edm::ParameterSet> watchers = iConfig.getParameter<std::vector<edm::ParameterSet>>("Watchers");
m_hasWatchers = !watchers.empty();

// TLS and watchers
initializeTLS();
int thisID = getThreadIndex();
if (m_hasWatchers) {
for (auto& watcher : m_tls->watchers) {
watcher->registerConsumes(iC);
}
}

if (m_LHCTransport) {
m_LHCToken = iC.consumes<edm::HepMCProduct>(edm::InputTag("LHCTransport"));
}
Expand All @@ -183,6 +201,11 @@ void RunManagerMTWorker::beginRun(edm::EventSetup const& es) {
if (m_pUseMagneticField) {
m_pMagField = &es.getData(m_MagField);
}
if (m_hasWatchers) {
for (auto& watcher : m_tls->watchers) {
watcher->beginRun(es);
}
}
}

void RunManagerMTWorker::endRun() {
Expand All @@ -207,26 +230,26 @@ void RunManagerMTWorker::initializeTLS() {
m_tls->registry->connect(*otherRegistry);
if (thisID > 0) {
throw edm::Exception(edm::errors::Configuration)
<< "RunManagerMTWorker::initializeTLS : "
<< "SimActivityRegistry service (i.e. visualization) is not supported for more than 1 thread. "
<< " \n If this use case is needed, RunManagerMTWorker has to be updated.";
}
}
if (m_hasWatchers) {
createWatchers(m_p, m_tls->registry.get(), m_tls->watchers, m_tls->producers);
}
m_hasWatchers = createWatchers(m_p, m_tls->registry.get(), m_tls->watchers, m_tls->producers, thisID);
}

void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm::EventSetup& es) {
if (m_tls->threadInitialized)
return;

G4Timer timer;
timer.Start();

// I guess everything initialized here should be in thread_local storage
initializeTLS();
if (m_tls->threadInitialized)
return;

int thisID = getThreadIndex();
edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMTWorker::initializeThread " << thisID << " is started";
edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMTWorker::initializeG4 in thread " << thisID << " is started";

// Initialize per-thread output
G4Threading::G4SetThreadId(thisID);
Expand All @@ -240,7 +263,7 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm:
new CustomUIsessionToFile(m_pCustomUIsession.getUntrackedParameter<std::string>("ThreadFile", ""), thisID);
} else {
throw edm::Exception(edm::errors::Configuration)
<< "Invalid value of CustomUIsession.Type '" << uitype
<< "RunManagerMTWorker::initializeG4: Invalid value of CustomUIsession.Type '" << uitype
<< "', valid are MessageLogger, MessageLoggerThreadPrefix, FilePerThread";
}

Expand Down Expand Up @@ -278,9 +301,10 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm:

std::string fieldFile = m_p.getUntrackedParameter<std::string>("FileNameField", "");
if (!fieldFile.empty()) {
std::call_once(applyOnce, [this]() { dumpMF = true; });
if (dumpMF) {
edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMTWorker: Dump magnetic field to file " << fieldFile;
std::call_once(applyOnce, [this]() { m_dumpMF = true; });
if (m_dumpMF) {
edm::LogVerbatim("SimG4CoreApplication")
<< "RunManagerMTWorker::InitializeG4: Dump magnetic field to file " << fieldFile;
DumpMagneticField(tM->GetFieldManager()->GetDetectorField(), fieldFile);
}
}
Expand All @@ -294,18 +318,18 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm:
m_tls->sensCaloDets.swap(sensDets.second);

edm::LogVerbatim("SimG4CoreApplication")
<< "RunManagerMTWorker: Sensitive Detectors are built in thread " << thisID << " found "
<< "RunManagerMTWorker::InitializeG4: Sensitive Detectors are built in thread " << thisID << " found "
<< m_tls->sensTkDets.size() << " Tk type SD, and " << m_tls->sensCaloDets.size() << " Calo type SD";

// Set the physics list for the worker, share from master
PhysicsList* physicsList = runManagerMaster->physicsListForWorker();

edm::LogVerbatim("SimG4CoreApplication")
<< "RunManagerMTWorker: start initialisation of PhysicsList for the thread " << thisID;
<< "RunManagerMTWorker::InitializeG4: start initialisation of PhysicsList for the thread " << thisID;

// Geant4 UI commands in PreInit state
if (!runManagerMaster->G4Commands().empty()) {
G4cout << "RunManagerMTWorker: Requested UI commands: " << G4endl;
G4cout << "RunManagerMTWorker::InitializeG4: Requested UI commands: " << G4endl;
for (const std::string& command : runManagerMaster->G4Commands()) {
G4cout << " " << command << G4endl;
G4UImanager::GetUIpointer()->ApplyCommand(command);
Expand All @@ -319,7 +343,7 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm:

if (!m_tls->kernel->RunInitialization()) {
throw edm::Exception(edm::errors::Configuration)
<< "RunManagerMTWorker: Geant4 kernel initialization failed in thread " << thisID;
<< "RunManagerMTWorker::InitializeG4: Geant4 kernel initialization failed in thread " << thisID;
}
//tell all interesting parties that we are beginning the job
BeginOfJob aBeginOfJob(&es);
Expand All @@ -337,11 +361,11 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm:
initializeUserActions();

G4StateManager::GetStateManager()->SetNewState(G4State_Idle);
m_tls->threadInitialized = true;

timer.Stop();
edm::LogVerbatim("SimG4CoreApplication")
<< "RunManagerMTWorker::initializeThread done for the thread " << thisID << " " << timer;
<< "RunManagerMTWorker::initializeG4 done for the thread " << thisID << " " << timer;
m_tls->threadInitialized = true;
}

void RunManagerMTWorker::initializeUserActions() {
Expand Down Expand Up @@ -453,7 +477,7 @@ std::unique_ptr<G4SimEvent> RunManagerMTWorker::produce(const edm::Event& inpevt
// Initialize run
if (inpevt.id().run() != m_tls->currentRunNumber) {
edm::LogVerbatim("SimG4CoreApplication")
<< "RunID= " << inpevt.id().run() << " TLS RunID= " << m_tls->currentRunNumber;
<< "RunManagerMTWorker::produce: RunID= " << inpevt.id().run() << " TLS RunID= " << m_tls->currentRunNumber;
if (m_tls->currentRunNumber != 0 && !m_tls->runTerminated) {
// If previous run in this thread was not terminated via endRun() call,
// terminate it now
Expand Down Expand Up @@ -573,7 +597,7 @@ void RunManagerMTWorker::DumpMagneticField(const G4Field* field, const std::stri
std::ofstream fout(file.c_str(), std::ios::out);
if (fout.fail()) {
edm::LogWarning("SimG4CoreApplication")
<< " RunManager WARNING : error opening file <" << file << "> for magnetic field";
<< "MTWorker::DumpMagneticField: error opening file <" << file << "> for magnetic field";
} else {
// CMS magnetic field volume
double rmax = 9000 * mm;
Expand Down
17 changes: 17 additions & 0 deletions SimG4Core/Watcher/interface/SimWatcher.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,30 @@ called by the dynamic loading code.
// Created: Tue Nov 22 15:35:11 EST 2005
//

#include "FWCore/Framework/interface/EventSetup.h"
#include "FWCore/Framework/interface/ConsumesCollector.h"

class SimWatcher {
public:
SimWatcher() {}
virtual ~SimWatcher() {}

// Two methods are needed to be implemented in the thread
// safe watchers and producers
virtual void registerConsumes(edm::ConsumesCollector){};
virtual void beginRun(edm::EventSetup const &){};

bool isMT() const { return applicableForMT; }

SimWatcher(const SimWatcher &) = delete;
const SimWatcher &operator=(const SimWatcher &) = delete;

protected:
// Set "true" for thread safe watchers/producers
void setMT(bool val) { applicableForMT = val; }

private:
bool applicableForMT{false};
};

#endif

0 comments on commit 5a0d088

Please sign in to comment.