Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove access to bare ROOT objects in HLTFiltersDQMonitor #40426

Merged
merged 4 commits into from
Jan 7, 2023
Merged

remove access to bare ROOT objects in HLTFiltersDQMonitor #40426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0f5ff99
added unit test for HLTFiltersDQMonitor plugin
missirol Jan 4, 2023
f51c546
removed access to bare ROOT objects in HLTFiltersDQMonitor
missirol Jan 4, 2023
6211e8b
renaming of cfi and parameters of HLTFiltersDQMonitor
missirol Jan 4, 2023
ed5f69b
avoid extra map searches in HLTFiltersDQMonitor
missirol Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 21 additions & 18 deletions DQMOffline/Trigger/plugins/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,30 @@
<use name="FWCore/Framework"/>
<use name="FWCore/ParameterSet"/>
<use name="FWCore/ServiceRegistry"/>
<use name="CommonTools/CandUtils"/>
<use name="CommonTools/Egamma"/>
<use name="CommonTools/TriggerUtils"/>
<use name="CommonTools/Utils"/>
<use name="DQMOffline/Trigger"/>
<use name="DQMServices/Core"/>
<use name="DataFormats/BTauReco"/>
<use name="DataFormats/Common"/>
<use name="DataFormats/EgammaCandidates"/>
<use name="DataFormats/HLTReco"/>
<use name="DataFormats/JetReco"/>
<use name="DataFormats/METReco"/>
<use name="DataFormats/MuonReco"/>
<use name="DataFormats/EgammaCandidates"/>
<use name="DataFormats/TrackReco"/>
<use name="DataFormats/BTauReco"/>
<use name="DataFormats/TauReco"/>
<use name="DataFormats/METReco"/>
<use name="DataFormats/JetReco"/>
<use name="CommonTools/Egamma"/>
<use name="RecoVertex/KalmanVertexFit"/>
<use name="HLTrigger/HLTcore"/>
<use name="DQMServices/Core"/>
<use name="CommonTools/CandUtils"/>
<use name="CommonTools/Utils"/>
<use name="CommonTools/TriggerUtils"/>
<use name="DataFormats/TrackReco"/>
<use name="DataFormats/VertexReco"/>
<use name="DQMOffline/Trigger"/>
<use name="FWCore/Framework"/>
<use name="FWCore/MessageLogger"/>
<use name="FWCore/ParameterSet"/>
<use name="FWCore/ServiceRegistry"/>
<use name="FWCore/Utilities"/>
<use name="HLTrigger/HLTcore"/>
<use name="RecoVertex/KalmanVertexFit"/>
<use name="TrackingTools/TransientTrack"/>
<use name="root"/>
<use name="roofit"/>
<use name="boost"/>
<use name="roofit"/>
<use name="root"/>
<library file="*.cc" name="DQMOfflineTriggerPlugins">
<flags EDM_PLUGIN="1"/>
</library>
658 changes: 373 additions & 285 deletions DQMOffline/Trigger/plugins/HLTFiltersDQMonitor.cc
View file
Open in desktop

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions DQMOffline/Trigger/python/DQMOffline_Trigger_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
from DQM.HLTEvF.HLTObjectsMonitor_cfi import *

# monitoring of efficiencies of HLT paths and filters
from DQMOffline.Trigger.hltFiltersDQMonitor_cfi import *
hltFiltersDQM = hltFiltersDQMonitor.clone(
folderName = 'HLT/Filters',
efficPlotNamePrefix = 'effic_',
triggerResults = 'TriggerResults::HLT',
triggerSummaryAOD = 'hltTriggerSummaryAOD::HLT',
triggerSummaryRAW = 'hltTriggerSummaryRAW::HLT',
from DQMOffline.Trigger.dqmHLTFiltersDQMonitor_cfi import dqmHLTFiltersDQMonitor as _dqmHLTFiltersDQMonitor
dqmHLTFiltersDQMonitor = _dqmHLTFiltersDQMonitor.clone(
folderName = 'HLT/Filters',
efficPlotNamePrefix = 'effic_',
triggerResults = 'TriggerResults::HLT',
triggerEvent = 'hltTriggerSummaryAOD::HLT',
triggerEventWithRefs = 'hltTriggerSummaryRAW::HLT'
)

# Lumi
Expand Down Expand Up @@ -121,7 +121,7 @@
## ADD here sequences/modules which rely ONLY on collections stored in the AOD format
offlineHLTSourceOnAOD = cms.Sequence(
dqmEnvHLT
* hltFiltersDQM
* dqmHLTFiltersDQMonitor
* lumiMonitorHLTsequence
* muonFullOfflineDQM
* HLTTauDQMOffline
Expand All @@ -145,7 +145,7 @@

## w/ the RECO step on-the-fly (to be added to offlineHLTSourceOnAOD which should run anyhow)
offlineHLTSourceWithRECO = cms.Sequence(
hltFiltersDQM
dqmHLTFiltersDQMonitor
* egHLTOffDQMSource ## NEEDED in VALIDATION, not really in MONITORING
* egHLTOffDQMSource_HEP17 ## NEEDED in VALIDATION, not really in MONITORING
* jetMETHLTOfflineAnalyzer
Expand Down
9 changes: 5 additions & 4 deletions DQMOffline/Trigger/python/DQMOffline_Trigger_cosmics_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#onlineHLTSource = cms.Sequence(EcalPi0Mon*EcalPhiSymMon*hltMonMuBits*dqmEnvHLTOnline)

# HLT Offline -----------------------------------
from DQMOffline.Trigger.hltFiltersDQMonitor_cfi import *
from DQMOffline.Trigger.dqmHLTFiltersDQMonitor_cfi import *

# EGamma
from DQMOffline.Trigger.EgHLTOfflineSource_cfi import *
Expand All @@ -51,14 +51,15 @@
dqmEnvHLT= DQMServices.Components.DQMEnvironment_cfi.dqmEnv.clone(
subSystemFolder = 'HLT'
)

offlineHLTSource = cms.Sequence(
hltFiltersDQMonitor *
dqmHLTFiltersDQMonitor *
egHLTOffDQMSource *
hltMuonOfflineAnalyzers *
HLTTauDQMOffline *
jetMETHLTOfflineSource *
dqmEnvHLT
)

#triggerCosmicOfflineDQMSource = cms.Sequence(onlineHLTSource*offlineHLTSource)
triggerCosmicOfflineDQMSource = cms.Sequence(offlineHLTSource)
#triggerCosmicOfflineDQMSource = cms.Sequence(onlineHLTSource*offlineHLTSource)
triggerCosmicOfflineDQMSource = cms.Sequence(offlineHLTSource)
8 changes: 4 additions & 4 deletions DQMOffline/Trigger/python/HLT_DQM_Offline_cff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import FWCore.ParameterSet.Config as cms

from DQMOffline.Trigger.hltFiltersDQMonitor_cfi import *
hltFiltersDQMonitor.triggerSummaryAOD = 'hltTriggerSummaryAOD::HLT'
hltFiltersDQMonitor.triggerResults = 'TriggerResults::HLT'
from DQMOffline.Trigger.dqmHLTFiltersDQMonitor_cfi import *
dqmHLTFiltersDQMonitor.triggerEvent = 'hltTriggerSummaryAOD::HLT'
dqmHLTFiltersDQMonitor.triggerResults = 'TriggerResults::HLT'

from DQMOffline.Trigger.HLTEventInfoClient_cfi import *

hltDqmOffline = cms.Sequence(hltFiltersDQMonitor*hltEventInfoClient)
hltDqmOffline = cms.Sequence( dqmHLTFiltersDQMonitor * hltEventInfoClient )
3 changes: 3 additions & 0 deletions DQMOffline/Trigger/test/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
<use name="DQMOffline/Trigger"/>
<use name="catch2"/>
</bin>

<!-- test the HLTFiltersDQMonitor plugin -->
<test name="testHLTFiltersDQMonitor" command="testHLTFiltersDQMonitor.sh"/>
4 changes: 2 additions & 2 deletions DQMOffline/Trigger/test/fv_dqmoffline_sourceclient-file_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,12 @@


###############################
# Only hltFiltersDQMonitor
# Only dqmHLTFiltersDQMonitor
#
##############################
#
# Offline
process.pHLT = cms.Path(process.hltFiltersDQMonitor)
process.pHLT = cms.Path(process.dqmHLTFiltersDQMonitor)



Expand Down
54 changes: 54 additions & 0 deletions DQMOffline/Trigger/test/harvesting_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import FWCore.ParameterSet.Config as cms

## CLI parser
import argparse
import sys

parser = argparse.ArgumentParser(
prog = 'cmsRun '+sys.argv[0]+' --',
description = 'Configuration file to run the DQMFileSaver on DQMIO input files.',
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)

parser.add_argument('-t', '--nThreads', type = int, help = 'Number of threads',
default = 4)

parser.add_argument('-s', '--nStreams', type = int, help = 'Number of EDM streams',
default = 0)

parser.add_argument('-i', '--inputFiles', nargs = '+', help = 'List of DQMIO input files',
default = ['file:DQMIO.root'])

argv = sys.argv[:]
if '--' in argv:
argv.remove('--')
args, unknown = parser.parse_known_args(argv)

# Process
process = cms.Process('HARVESTING')

process.options.numberOfThreads = args.nThreads
process.options.numberOfStreams = args.nStreams
process.options.numberOfConcurrentLuminosityBlocks = 1

# Source (DQM input)
process.source = cms.Source('DQMRootSource',
fileNames = cms.untracked.vstring(args.inputFiles)
)

# DQMStore (Service)
process.load('DQMServices.Core.DQMStore_cfi')

# MessageLogger (Service)
process.load('FWCore.MessageLogger.MessageLogger_cfi')

# Output module (file in ROOT format)
from DQMServices.Components.DQMFileSaver_cfi import dqmSaver as _dqmSaver
process.dqmSaver = _dqmSaver.clone(
workflow = '/DQMOffline/Trigger/'+process.name_()
)

# EndPath
process.endp = cms.EndPath(
process.dqmSaver
)
30 changes: 30 additions & 0 deletions DQMOffline/Trigger/test/readme.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
Unit test: `testHLTFiltersDQMonitor`
------------------------------------

Test of the DQM plugin `HLTFiltersDQMonitor`.

- To run the test via `scram`
```sh
scram build runtests_testHLTFiltersDQMonitor
```

- To run the test without `scram`
```sh
LOCALTOP="${CMSSW_BASE}" "${CMSSW_BASE}"/src/DQMOffline/Trigger/test/testHLTFiltersDQMonitor.sh
```

- To show info on command-line arguments of `testHLTFiltersDQMonitor_cfg.py`
```sh
python3 "${CMSSW_BASE}"/src/DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py -h
```

- To execute cmsRun with `testHLTFiltersDQMonitor_cfg.py` (example)
```sh
cmsRun "${CMSSW_BASE}"/src/DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py -- -t 4 -s 0 -o tmp.root -n 100
```

- To create a bare ROOT file from the DQMIO output of `testHLTFiltersDQMonitor_cfg.py`,
run the harvesting step as follows
```sh
cmsRun "${CMSSW_BASE}"/src/DQMOffline/Trigger/test/harvesting_cfg.py -- -i file:tmp.root
```
13 changes: 13 additions & 0 deletions DQMOffline/Trigger/test/testHLTFiltersDQMonitor.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

# Pass in name and status
function die {
printf "\n%s: status %s\n" "$1" "$2"
exit $2
}

# run test job
TESTDIR="${LOCALTOP}"/src/DQMOffline/Trigger/test

cmsRun "${TESTDIR}"/testHLTFiltersDQMonitor_cfg.py -- -t 4 -n 128 \
|| die "Failure running testHLTFiltersDQMonitor_cfg.py" $?
105 changes: 105 additions & 0 deletions DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import FWCore.ParameterSet.Config as cms

## CLI parser
import argparse
import sys

parser = argparse.ArgumentParser(
prog = 'cmsRun '+sys.argv[0]+' --',
description = 'Configuration file to test of the HLTFiltersDQMonitor plugin.',
formatter_class = argparse.ArgumentDefaultsHelpFormatter
)

Comment on lines +3 to +12
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for completeness: this argparse configuration was ~copied from

cmssw/FWCore/Integration/test/testSwitchProducerTask_cfg.py

Lines 3 to 14 in 287f4db

import argparse
import sys
parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test SwitchProducer in Task.')
parser.add_argument("--disableTest2", help="Disable test2 SwitchProducer case", action="store_true")
parser.add_argument("--input", help="Read input file from a previous step of this same configuration", action="store_true")
parser.add_argument("--conditionalTask", help="Use ConditionalTask instead of Task", action="store_true")
argv = sys.argv[:]
if '--' in argv:
argv.remove("--")
args, unknown = parser.parse_known_args(argv)

I was wondering if there are cases where VarParsing has advantages (or, is recommended) wrt argparse.

The only hiccup I encountered with this argparse was with

cmsRun DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py -- -h

which I naively expected to work, but leads to an error [1] (didn't figure out the solution).

[1]

----- Begin Fatal Exception 04-Jan-2023 19:08:48 CET-----------------------
An exception of category 'ConfigFileReadError' occurred while
   [0] Processing the python configuration file named DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py
Exception Message:
 unknown python problem occurred.
SystemExit: 0

At:
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(2569): exit
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(1100): __call__
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(1935): take_action
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(2007): consume_optional
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(2067): _parse_known_args
  /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc11/external/python3/3.9.14-76a14295dd5255228210eb596893b98c/lib/python3.9/argparse.py(1861): parse_known_args
  DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py(37): <module>

----- End Fatal Exception -------------------------------------------------

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for completeness: this argparse configuration was ~copied from

For future reference the use of argparse is also described in
https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideAboutPythonConfigFile#argparse

The only hiccup I encountered with this argparse was with

cmsRun DQMOffline/Trigger/test/testHLTFiltersDQMonitor_cfg.py -- -h

which I naively expected to work, but leads to an error [1]

Hah, interesting. It seems that sys.exit(0) gets communicated as a thrown exception by PyBind11 in the C++ side, which framework then reports as shown (the printout from -h is there, although in a slightly confusing way). It is not immediately clear to me if we should do something about it, so a specific issue might be worth it.

As for VarParsing vs argparse, core does not have a specific recommendation. First has longer history within CMSSW (and not really developed further), and the latter comes from python's standard library.

parser.add_argument('-t', '--nThreads', type = int, help = 'Number of threads',
default = 4)

parser.add_argument('-s', '--nStreams', type = int, help = 'Number of EDM streams',
default = 0)

parser.add_argument('-i', '--inputFiles', nargs = '+', help = 'List of EDM input files',
default = ['/store/relval/CMSSW_12_6_0_pre2/RelValTTbar_14TeV/GEN-SIM-DIGI-RAW/125X_mcRun3_2022_realistic_v3-v1/2580000/2d96539c-b321-401f-b7b2-51884a5d421f.root'])

parser.add_argument('-n', '--maxEvents', type = int, help = 'Number of input events',
default = 100)

parser.add_argument('-o', '--outputFile', type = str, help = 'Path to output file in DQMIO format',
default = 'DQMIO.root')

parser.add_argument('--wantSummary', action = 'store_true', help = 'Value of process.options.wantSummary',
default = False)

parser.add_argument('-d', '--debugMode', action = 'store_true', help = 'Enable debug info (requires recompiling first with \'USER_CXXFLAGS="-DEDM_ML_DEBUG" scram b\')',
default = False)

argv = sys.argv[:]
if '--' in argv:
argv.remove('--')
args, unknown = parser.parse_known_args(argv)

## Process
process = cms.Process('TEST')

process.options.numberOfThreads = args.nThreads
process.options.numberOfStreams = args.nStreams
process.options.wantSummary = args.wantSummary
process.maxEvents.input = args.maxEvents

## Source
process.source = cms.Source('PoolSource',
fileNames = cms.untracked.vstring(args.inputFiles),
inputCommands = cms.untracked.vstring(
'drop *',
'keep edmTriggerResults_*_*_*',
'keep triggerTriggerEvent_*_*_*',
'keep triggerTriggerEventWithRefs_*_*_*'
)
)

## MessageLogger (Service)
process.load('FWCore.MessageLogger.MessageLogger_cfi')
process.MessageLogger.cerr.FwkReport.reportEvery = 1 # only report every Nth event start
process.MessageLogger.cerr.FwkReport.limit = -1 # max number of reported messages (all if -1)
process.MessageLogger.cerr.enableStatistics = False # enable "MessageLogger Summary" message

## DQMStore (Service)
process.load('DQMServices.Core.DQMStore_cfi')

## FastTimerService (Service)
from HLTrigger.Timer.FastTimerService_cfi import FastTimerService as _FastTimerService
process.FastTimerService = _FastTimerService.clone(
enableDQM = False,
printEventSummary = False,
printJobSummary = True,
printRunSummary = False,
writeJSONSummary = False
)
process.MessageLogger.FastReport = dict()

## EventData Modules
from DQMOffline.Trigger.dqmHLTFiltersDQMonitor_cfi import dqmHLTFiltersDQMonitor as _dqmHLTFiltersDQMonitor
process.dqmHLTFiltersDQMonitor = _dqmHLTFiltersDQMonitor.clone(
folderName = 'HLT/Filters',
efficPlotNamePrefix = 'effic_',
triggerResults = 'TriggerResults::HLT',
triggerEvent = 'hltTriggerSummaryAOD::HLT',
triggerEventWithRefs = 'hltTriggerSummaryRAW::HLT'
)
process.MessageLogger.HLTFiltersDQMonitor = dict()
if args.debugMode:
process.MessageLogger.cerr.threshold = 'DEBUG'
process.MessageLogger.debugModules = ['dqmHLTFiltersDQMonitor']

## Output Modules
process.dqmOutput = cms.OutputModule('DQMRootOutputModule',
fileName = cms.untracked.string(args.outputFile)
)

## Path
process.testPath = cms.Path(
process.dqmHLTFiltersDQMonitor
)

## EndPath
process.testEndPath = cms.EndPath(
process.dqmOutput
)
4 changes: 2 additions & 2 deletions DQMOffline/Trigger/test/triggerSequenceTest_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
)


process.p = cms.EndPath(process.hltFiltersDQMonitor)
process.p = cms.EndPath(process.dqmHLTFiltersDQMonitor)

process.pp = cms.Path(process.dqmEnv+process.dqmSaver)
process.DQMStore.verbose = 0
Expand Down Expand Up @@ -92,7 +92,7 @@
#
# # copy stdout to a file
# process.MessageLogger.detailedInfo = process.MessageLogger.cout
# process.MessageLogger.debugModules = ['hltFiltersDQMonitor']
# process.MessageLogger.debugModules = ['dqmHLTFiltersDQMonitor']
# process.MessageLogger.critical = cms.untracked.PSet(
# threshold = cms.untracked.string('ERROR'),
# #threshold = cms.untracked.string('INFO'),
Expand Down
4 changes: 2 additions & 2 deletions DQMOffline/Trigger/test/trigger_dqmoffline_cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@
critical = cms.untracked.PSet(
threshold = cms.untracked.string('ERROR')
),
debugModules = cms.untracked.vstring('hltFiltersDQMonitor'),
debugModules = cms.untracked.vstring('dqmHLTFiltersDQMonitor'),
cout = cms.untracked.PSet(
threshold = cms.untracked.string('WARNING'),
WARNING = cms.untracked.PSet(
Expand All @@ -132,7 +132,7 @@
process.allPath = cms.Path( process.triggerCosmicOfflineDQMSource * process.triggerOfflineDQMClient * process.hltOfflineDQMClient * process.dqmStoreStats )
#process.allPath = cms.Path( process.triggerCosmicOfflineDQMSource*process.hltOfflineDQMClient)
#process.allPath = cms.Path( process.DQMOfflineCosmics)
#process.psource = cms.Path(process.hltFiltersDQMonitor)
#process.psource = cms.Path(process.dqmHLTFiltersDQMonitor)

process.p = cms.EndPath(process.dqmSaver)
process.DQMStore.verbose = 0
Expand Down
Loading