From 6dff24d4f78a4bc987bec52deacdcdfe11ee9642 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Mon, 23 Aug 2021 14:47:38 +0200 Subject: [PATCH 1/8] Plugins and test config for NanoAOD-based calibtree from SiStripCalCosmics ALCARECO --- CalibTracker/SiStripCommon/BuildFile.xml | 3 + .../SiStripOnTrackClusterTableProducerBase.h | 56 ++++++ .../SiStripCommon/plugins/BuildFile.xml | 1 + .../SiStripCommon/plugins/SealModules.cc | 4 + ...SiStripLorentzAngleRunInfoTableProducer.cc | 107 +++++++++++ ...SiStripPositionCorrectionsTableProducer.cc | 75 ++++++++ .../SiStripOnTrackClusterTableProducerBase.cc | 68 +++++++ .../produceCalibrationTree_CosmicsLABP_cfg.py | 173 ++++++++++++++++++ 8 files changed, 487 insertions(+) create mode 100644 CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h create mode 100644 CalibTracker/SiStripCommon/plugins/SiStripLorentzAngleRunInfoTableProducer.cc create mode 100644 CalibTracker/SiStripCommon/plugins/SiStripPositionCorrectionsTableProducer.cc create mode 100644 CalibTracker/SiStripCommon/src/SiStripOnTrackClusterTableProducerBase.cc create mode 100644 CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py diff --git a/CalibTracker/SiStripCommon/BuildFile.xml b/CalibTracker/SiStripCommon/BuildFile.xml index 17b87a3ad87ae..dba91db4041f4 100644 --- a/CalibTracker/SiStripCommon/BuildFile.xml +++ b/CalibTracker/SiStripCommon/BuildFile.xml @@ -2,8 +2,11 @@ + + + diff --git a/CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h b/CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h new file mode 100644 index 0000000000000..4dec2d9591a6c --- /dev/null +++ b/CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h @@ -0,0 +1,56 @@ +#include "FWCore/Framework/interface/stream/EDProducer.h" +#include "FWCore/Framework/interface/ConsumesCollector.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +#include "DataFormats/NanoAOD/interface/FlatTable.h" +#include "DataFormats/SiStripCluster/interface/SiStripCluster.h" +#include "DataFormats/TrackReco/interface/Track.h" +#include "TrackingTools/PatternTools/interface/TrajTrackAssociation.h" + +class SiStripOnTrackClusterTableProducerBase : public edm::stream::EDProducer<> { +public: + explicit SiStripOnTrackClusterTableProducerBase(const edm::ParameterSet& params) + : m_name(params.getParameter("name")), + m_doc(params.existsAs("doc") ? params.getParameter("doc") : ""), + m_extension(params.existsAs("extension") ? params.getParameter("extension") : true), + m_tracks_token(consumes>(params.getParameter("Tracks"))), + m_association_token(consumes(params.getParameter("Tracks"))) { + produces(); + } + ~SiStripOnTrackClusterTableProducerBase() override; + + void produce(edm::Event& iEvent, const edm::EventSetup& iSetup) final; + + struct OnTrackCluster { + uint32_t det; + const SiStripCluster* cluster; + const Trajectory* traj; + const reco::Track* track; + const TrajectoryMeasurement& measurement; + OnTrackCluster(uint32_t detId, + const SiStripCluster* stripCluster, + const Trajectory* trajectory, + const reco::Track* track_, + const TrajectoryMeasurement& measurement_) + : det{detId}, cluster{stripCluster}, traj{trajectory}, track{track_}, measurement{measurement_} {} + }; + + virtual void fillTable(const std::vector& clusters, + const edm::View& tracks, + nanoaod::FlatTable* table, + const edm::EventSetup& iSetup) = 0; + + template + static void addColumn(nanoaod::FlatTable* table, const std::string& name, VALUES&& values, const std::string& doc) { + using value_type = typename std::remove_reference::type::value_type; + table->template addColumn(name, values, doc); + } + +private: + const std::string m_name; + const std::string m_doc; + bool m_extension; + + const edm::EDGetTokenT> m_tracks_token; + const edm::EDGetTokenT m_association_token; +}; diff --git a/CalibTracker/SiStripCommon/plugins/BuildFile.xml b/CalibTracker/SiStripCommon/plugins/BuildFile.xml index 039a07453e59c..64a3dcae11884 100644 --- a/CalibTracker/SiStripCommon/plugins/BuildFile.xml +++ b/CalibTracker/SiStripCommon/plugins/BuildFile.xml @@ -9,6 +9,7 @@ + diff --git a/CalibTracker/SiStripCommon/plugins/SealModules.cc b/CalibTracker/SiStripCommon/plugins/SealModules.cc index 6613d45b91294..e700bb6a6472c 100644 --- a/CalibTracker/SiStripCommon/plugins/SealModules.cc +++ b/CalibTracker/SiStripCommon/plugins/SealModules.cc @@ -24,3 +24,7 @@ DEFINE_FWK_MODULE(ShallowSimhitClustersProducer); DEFINE_FWK_MODULE(ShallowTracksProducer); DEFINE_FWK_MODULE(ShallowSimTracksProducer); DEFINE_FWK_MODULE(ShallowGainCalibration); + +#include "PhysicsTools/NanoAOD/interface/SimpleFlatTableProducer.h" +typedef SimpleFlatTableProducer SimpleTrackFlatTableProducer; +DEFINE_FWK_MODULE(SimpleTrackFlatTableProducer); diff --git a/CalibTracker/SiStripCommon/plugins/SiStripLorentzAngleRunInfoTableProducer.cc b/CalibTracker/SiStripCommon/plugins/SiStripLorentzAngleRunInfoTableProducer.cc new file mode 100644 index 0000000000000..7e09ef4f58a7a --- /dev/null +++ b/CalibTracker/SiStripCommon/plugins/SiStripLorentzAngleRunInfoTableProducer.cc @@ -0,0 +1,107 @@ +#include "FWCore/Framework/interface/global/EDProducer.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Framework/interface/ConsumesCollector.h" +#include "DataFormats/NanoAOD/interface/FlatTable.h" + +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" + +#include "MagneticField/Engine/interface/MagneticField.h" +#include "MagneticField/Records/interface/IdealMagneticFieldRecord.h" +#include "CondFormats/SiStripObjects/interface/SiStripLorentzAngle.h" +#include "CalibTracker/Records/interface/SiStripDependentRecords.h" +#include "Geometry/TrackerGeometryBuilder/interface/StripGeomDetUnit.h" +#include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h" +#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h" + +#include "CalibTracker/SiStripCommon/interface/ShallowTools.h" + +class SiStripLorentzAngleRunInfoTableProducer : public edm::global::EDProducer { +public: + explicit SiStripLorentzAngleRunInfoTableProducer(const edm::ParameterSet& params) + : m_name{params.getParameter("name")}, + m_magFieldName{params.getParameter("magFieldName")}, + m_doc{params.existsAs("doc") ? params.getParameter("doc") : ""}, + m_tkGeomToken{esConsumes()}, + m_magFieldToken{esConsumes()}, + m_lorentzAngleToken{esConsumes()} { + produces(); + produces("magField"); + } + + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add("name", "Det"); + desc.add("magFieldName", "magField"); + desc.add("doc", "Run info for the Lorentz angle measurement"); + descriptions.add("siStripLorentzAngleRunInfoTable", desc); + } + + void globalBeginRunProduce(edm::Run& iRun, edm::EventSetup const& iSetup) const override; + +private: + const std::string m_name, m_magFieldName; + const std::string m_doc; + edm::ESGetToken m_tkGeomToken; + edm::ESGetToken m_magFieldToken; + edm::ESGetToken m_lorentzAngleToken; +}; + +namespace { + template + void addColumn(nanoaod::FlatTable* table, const std::string& name, VALUES&& values, const std::string& doc) { + using value_type = typename std::remove_reference::type::value_type; + table->template addColumn(name, values, doc); + } +} // namespace + +void SiStripLorentzAngleRunInfoTableProducer::globalBeginRunProduce(edm::Run& iRun, + edm::EventSetup const& iSetup) const { + const auto& tkGeom = iSetup.getData(m_tkGeomToken); + const auto& magField = iSetup.getData(m_magFieldToken); + const auto& lorentzAngle = iSetup.getData(m_lorentzAngleToken); + std::vector c_rawid; + std::vector c_globalZofunitlocalY, c_localB, c_BdotY, c_driftx, c_drifty, c_driftz, c_lorentzAngle; + + auto dets = tkGeom.detsTIB(); + dets.insert(dets.end(), tkGeom.detsTID().begin(), tkGeom.detsTID().end()); + dets.insert(dets.end(), tkGeom.detsTOB().begin(), tkGeom.detsTOB().end()); + dets.insert(dets.end(), tkGeom.detsTEC().begin(), tkGeom.detsTEC().end()); + for (auto det : dets) { + auto detid = det->geographicalId().rawId(); + const StripGeomDetUnit* stripDet = dynamic_cast(tkGeom.idToDet(det->geographicalId())); + if (stripDet) { + c_rawid.push_back(detid); + c_globalZofunitlocalY.push_back(stripDet->toGlobal(LocalVector(0, 1, 0)).z()); + const auto locB = magField.inTesla(stripDet->surface().position()); + c_localB.push_back(locB.mag()); + c_BdotY.push_back(stripDet->surface().toLocal(locB).y()); + const auto drift = shallow::drift(stripDet, magField, lorentzAngle); + c_driftx.push_back(drift.x()); + c_drifty.push_back(drift.y()); + c_driftz.push_back(drift.z()); + c_lorentzAngle.push_back(lorentzAngle.getLorentzAngle(detid)); + } + } + auto out = std::make_unique(c_rawid.size(), m_name, false, false); + addColumn(out.get(), "rawid", c_rawid, "DetId"); + addColumn(out.get(), "globalZofunitlocalY", c_globalZofunitlocalY, "z component of a local unit vector along y"); + addColumn(out.get(), "localB", c_localB, "Local magnitude of the magnetic field"); + addColumn(out.get(), "BdotY", c_BdotY, "Magnetic field projection on the local y axis"); + addColumn(out.get(), "driftx", c_driftx, "x component of the drift vector"); + addColumn(out.get(), "drifty", c_drifty, "y component of the drift vector"); + addColumn(out.get(), "driftz", c_driftz, "z component of the drift vector"); + addColumn(out.get(), "lorentzAngle", c_lorentzAngle, "Lorentz angle from database"); + iRun.put(std::move(out)); + + auto out2 = std::make_unique(1, m_magFieldName, true, false); + out2->addColumnValue( + "origin", magField.inTesla(GlobalPoint(0, 0, 0)).z(), "z-component of the magnetic field at (0,0,0) in Tesla"); + iRun.put(std::move(out2), "magField"); +} + +#include "FWCore/PluginManager/interface/ModuleDef.h" +#include "FWCore/Framework/interface/MakerMacros.h" +DEFINE_FWK_MODULE(SiStripLorentzAngleRunInfoTableProducer); diff --git a/CalibTracker/SiStripCommon/plugins/SiStripPositionCorrectionsTableProducer.cc b/CalibTracker/SiStripCommon/plugins/SiStripPositionCorrectionsTableProducer.cc new file mode 100644 index 0000000000000..1f7d93008807f --- /dev/null +++ b/CalibTracker/SiStripCommon/plugins/SiStripPositionCorrectionsTableProducer.cc @@ -0,0 +1,75 @@ +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" + +#include "Geometry/TrackerGeometryBuilder/interface/StripGeomDetUnit.h" +#include "Geometry/TrackerGeometryBuilder/interface/TrackerGeometry.h" +#include "Geometry/Records/interface/TrackerDigiGeometryRecord.h" + +#include "RecoLocalTracker/SiStripClusterizer/interface/SiStripClusterInfo.h" + +#include "CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h" + +class SiStripPositionCorrectionsTableProducer : public SiStripOnTrackClusterTableProducerBase { +public: + explicit SiStripPositionCorrectionsTableProducer(const edm::ParameterSet& params) + : SiStripOnTrackClusterTableProducerBase(params), + m_clusterInfo(consumesCollector()), + m_tkGeomToken{esConsumes<>()} {} + + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add("name", "cluster"); + desc.add("doc", "On-track cluster properties for Lorentz angle and backplane correction measurement"); + desc.add("extension", false); + desc.add("Tracks", edm::InputTag{"generalTracks"}); + descriptions.add("siStripPositionCorrectionsTable", desc); + } + + void fillTable(const std::vector& clusters, + const edm::View& tracks, + nanoaod::FlatTable* table, + const edm::EventSetup& iSetup) final; + +private: + SiStripClusterInfo m_clusterInfo; + edm::ESGetToken m_tkGeomToken; +}; + +void SiStripPositionCorrectionsTableProducer::fillTable(const std::vector& clusters, + const edm::View& tracks, + nanoaod::FlatTable* table, + const edm::EventSetup& iSetup) { + const auto& tkGeom = iSetup.getData(m_tkGeomToken); + std::vector c_nstrips; + std::vector c_barycenter, c_variance, c_localdirx, c_localdiry, c_localdirz, c_localx, c_rhlocalx, + c_rhlocalxerr; + for (const auto clus : clusters) { + c_nstrips.push_back(clus.cluster->amplitudes().size()); + m_clusterInfo.setCluster(*clus.cluster, clus.det); + c_variance.push_back(m_clusterInfo.variance()); + const auto& trajState = clus.measurement.updatedState(); + const auto trackDir = trajState.localDirection(); + c_localdirx.push_back(trackDir.x()); + c_localdiry.push_back(trackDir.y()); + c_localdirz.push_back(trackDir.z()); + const auto hit = clus.measurement.recHit()->hit(); + const auto stripDet = dynamic_cast(tkGeom.idToDet(hit->geographicalId())); + c_barycenter.push_back(stripDet->specificTopology().localPosition(clus.cluster->barycenter()).x()); + c_localx.push_back(stripDet->toLocal(trajState.globalPosition()).x()); + c_rhlocalx.push_back(hit->localPosition().x()); + c_rhlocalxerr.push_back(hit->localPositionError().xx()); + } + addColumn(table, "nstrips", c_nstrips, "cluster width"); + addColumn(table, "variance", c_variance, "Cluster variance"); + addColumn(table, "localdirx", c_localdirx, "x component of the local track direction"); + addColumn(table, "localdiry", c_localdiry, "y component of the local track direction"); + addColumn(table, "localdirz", c_localdirz, "z component of the local track direction"); + addColumn(table, "barycenter", c_barycenter, "Cluster barycenter (local x without corrections)"); + addColumn(table, "localx", c_localx, "Track local x"); + addColumn(table, "rhlocalx", c_rhlocalx, "RecHit local x"); + addColumn(table, "rhlocalxerr", c_rhlocalxerr, "RecHit local x uncertainty"); +} + +#include "FWCore/PluginManager/interface/ModuleDef.h" +#include "FWCore/Framework/interface/MakerMacros.h" +DEFINE_FWK_MODULE(SiStripPositionCorrectionsTableProducer); diff --git a/CalibTracker/SiStripCommon/src/SiStripOnTrackClusterTableProducerBase.cc b/CalibTracker/SiStripCommon/src/SiStripOnTrackClusterTableProducerBase.cc new file mode 100644 index 0000000000000..36e8804a3e88e --- /dev/null +++ b/CalibTracker/SiStripCommon/src/SiStripOnTrackClusterTableProducerBase.cc @@ -0,0 +1,68 @@ +#include "DataFormats/TrackerRecHit2D/interface/SiStripMatchedRecHit2D.h" +#include "DataFormats/TrackerRecHit2D/interface/SiStripRecHit2D.h" +#include "DataFormats/TrackerRecHit2D/interface/SiStripRecHit1D.h" + +#include "CalibTracker/SiStripCommon/interface/SiStripOnTrackClusterTableProducerBase.h" + +SiStripOnTrackClusterTableProducerBase::~SiStripOnTrackClusterTableProducerBase() {} + +namespace { + int findTrackIndex(const edm::View& tracks, const reco::Track* track) { + for (auto iTr = tracks.begin(); iTr != tracks.end(); ++iTr) { + if (&(*iTr) == track) { + return iTr - tracks.begin(); + } + } + return -2; + } +} // namespace + +void SiStripOnTrackClusterTableProducerBase::produce(edm::Event& iEvent, const edm::EventSetup& iSetup) { + edm::Handle> tracks; + iEvent.getByToken(m_tracks_token, tracks); + edm::Handle trajTrackAssociations; + iEvent.getByToken(m_association_token, trajTrackAssociations); + + std::vector clusters{}; + + for (const auto& assoc : *trajTrackAssociations) { + const auto traj = assoc.key.get(); + const auto track = assoc.val.get(); + + for (const auto& meas : traj->measurements()) { + const auto& trajState = meas.updatedState(); + if (!trajState.isValid()) + continue; + + // there can be 2 (stereo module), 1 (no stereo module), or 0 (no strip hit) clusters per measurement + const auto trechit = meas.recHit()->hit(); + const auto simple1d = dynamic_cast(trechit); + const auto simple = dynamic_cast(trechit); + const auto matched = dynamic_cast(trechit); + if (matched) { + clusters.emplace_back(matched->monoId(), &matched->monoCluster(), traj, track, meas); + clusters.emplace_back(matched->stereoId(), &matched->stereoCluster(), traj, track, meas); + } else if (simple) { + clusters.emplace_back(simple->geographicalId().rawId(), simple->cluster().get(), traj, track, meas); + } else if (simple1d) { + clusters.emplace_back(simple1d->geographicalId().rawId(), simple1d->cluster().get(), traj, track, meas); + } + } + } + + auto out = std::make_unique(clusters.size(), m_name, false, m_extension); + if (!m_extension) { + std::vector c_trackindex; + c_trackindex.reserve(clusters.size()); + std::vector c_rawid; + c_rawid.reserve(clusters.size()); + for (const auto clus : clusters) { + c_trackindex.push_back(findTrackIndex(*tracks, clus.track)); + c_rawid.push_back(clus.det); + } + addColumn(out.get(), "trackindex", c_trackindex, "Track index"); + addColumn(out.get(), "rawid", c_rawid, "DetId"); + } + fillTable(clusters, *tracks, out.get(), iSetup); + iEvent.put(std::move(out)); +} diff --git a/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py b/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py new file mode 100644 index 0000000000000..6341d7b945525 --- /dev/null +++ b/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py @@ -0,0 +1,173 @@ +from __future__ import print_function +import FWCore.ParameterSet.Config as cms +from FWCore.ParameterSet.VarParsing import VarParsing + +################################################################### +# Setup 'standard' options +################################################################### +options = VarParsing() + +options.register('conditionGT', + "auto:run2_data", + VarParsing.multiplicity.singleton, + VarParsing.varType.string, + "condition global tag for the job (\"auto:run2_data\" is default)") + +options.register('conditionOverwrite', + "", + VarParsing.multiplicity.singleton, + VarParsing.varType.string, + "configuration to overwrite the condition into the GT (\"\" is default)") + +options.register('inputCollection', + "ALCARECOSiStripCalMinBias", + VarParsing.multiplicity.singleton, + VarParsing.varType.string, + "collections to be used for input (\"ALCARECOSiStripCalMinBias\" is default, use 'generalTracks' for prompt reco and 'ctfWithMaterialTracksP5' for cosmic reco)") + +options.register('outputFile', + "calibTreeTest.root", + VarParsing.multiplicity.singleton, + VarParsing.varType.string, + "name for the output root file (\"calibTreeTest.root\" is default)") + +options.register('inputFiles', + '/store/data/Run2018D/Cosmics/ALCARECO/SiStripCalCosmics-UL18-v1/40000/0346DCE4-0C70-1344-A7EB-D488B627208C.root', + VarParsing.multiplicity.list, + VarParsing.varType.string, + "file to process") + +options.register('maxEvents', + -1, + VarParsing.multiplicity.singleton, + VarParsing.varType.int, + "number of events to process (\"-1\" for all)") + +options.register('runNumber', + -1, + VarParsing.multiplicity.singleton, + VarParsing.varType.int, + "run number to process (\"-1\" for all)") + +options.register('cosmicTriggers', '', + VarParsing.multiplicity.list, + VarParsing.varType.string, + 'cosmic triggers') + +options.parseArguments() +################################################################### +# To use the prompt reco dataset please use 'generalTracks' as inputCollection +# To use the cosmic reco dataset please use 'ctfWithMaterialTracksP5' as inputCollection + + +process = cms.Process('CALIB') +process.load('Configuration/StandardSequences/MagneticField_cff') +process.load('Configuration.Geometry.GeometryRecoDB_cff') +process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') + +from Configuration.AlCa.GlobalTag import GlobalTag +process.GlobalTag = GlobalTag(process.GlobalTag, options.conditionGT, options.conditionOverwrite) + +process.load('FWCore.MessageService.MessageLogger_cfi') +process.load('Configuration.StandardSequences.Services_cff') + +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(options.maxEvents)) + +process.source = cms.Source("PoolSource", fileNames=cms.untracked.vstring(options.inputFiles)) +if options.runNumber != -1: + if 'Cosmics' not in options.inputCollection: # FIXME: this should be improved + print("Restricting to the following events :") + print('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) + process.source.eventsToProcess = cms.untracked.VEventRange('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) + else: + print("Restricting to the following lumis for Cosmic runs only:") + print('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) + process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) + +process.options = cms.untracked.PSet( + wantSummary = cms.untracked.bool(True) + ) +process.MessageLogger.cerr.FwkReport.reportEvery = 10000 + +inTracks = cms.InputTag(options.inputCollection) + +process.load('CalibTracker.SiStripCommon.prescaleEvent_cfi') +process.load('CalibTracker.Configuration.Filter_Refit_cff') +## use CalibrationTracks (for clusters) and CalibrationTracksRefit (for tracks) +process.CalibrationTracks.src = inTracks +tracksForCalib = cms.InputTag("CalibrationTracksRefit") + +process.prescaleEvent.prescale = 1 +process.load("CalibTracker.SiStripCommon.SiStripBFieldFilter_cfi") + +from HLTrigger.HLTfilters.triggerResultsFilter_cfi import triggerResultsFilter + +process.IsolatedMuonFilter = triggerResultsFilter.clone( + triggerConditions = cms.vstring("HLT_IsoMu20_*"), + hltResults = cms.InputTag("TriggerResults", "", "HLT"), + l1tResults = cms.InputTag(""), + throw = cms.bool(False) + ) +if len(options.cosmicTriggers) > 0: + print("Cosmic triggers: {0}".format(", ".join(options.cosmicTriggers))) + process.IsolatedMuonFilter.triggerConditions = cms.vstring(options.cosmicTriggers) +else: + print("Cosmic triggers: {0} (default)".format(", ".join(process.IsolatedMuonFilter.triggerConditions))) + print("Argument passed: {0}".format(options.cosmicTriggers)) + +process.TkCalSeq = cms.Sequence( + process.prescaleEvent* + process.IsolatedMuonFilter* + process.siStripBFieldOnFilter* + process.CalibrationTracks, + cms.Task(process.MeasurementTrackerEvent), + cms.Task(process.offlineBeamSpot), + cms.Task(process.CalibrationTracksRefit) + ) + +process.load("PhysicsTools.NanoAOD.nano_cff") +process.load("PhysicsTools.NanoAOD.NanoAODEDMEventContent_cff") + +## as a test: it should be possible to add tracks fully at configuration level (+ declaring the plugin) +from PhysicsTools.NanoAOD.common_cff import * +## this is equivalent to ShallowTrackProducer as configured for the gain calibration +process.tracksTable = cms.EDProducer("SimpleTrackFlatTableProducer", + src=tracksForCalib, + cut=cms.string(""), + name=cms.string("track"), + doc=cms.string("SiStripCalMinBias ALCARECO tracks"), + singleton=cms.bool(False), + extension=cms.bool(False), + variables=cms.PSet( + chi2ndof=Var("chi2()/ndof", float), + pt=Var("pt()", float), + hitsvalid=Var("numberOfValidHits()", int), ## unsigned? + phi=Var("phi()", float), + eta=Var("eta()", float), + ) + ) +process.load("CalibTracker.SiStripCommon.siStripPositionCorrectionsTable_cfi") +process.siStripPositionCorrectionsTable.Tracks = tracksForCalib +process.load("CalibTracker.SiStripCommon.siStripLorentzAngleRunInfoTable_cfi") + +siStripCalCosmicsNanoTables = cms.Task( + process.nanoMetadata, + process.tracksTable, + process.siStripPositionCorrectionsTable, + process.siStripLorentzAngleRunInfoTable + ) + +process.nanoCTPath = cms.Path(process.TkCalSeq, siStripCalCosmicsNanoTables) + +process.out = cms.OutputModule("NanoAODOutputModule", + fileName=cms.untracked.string(options.outputFile), + outputCommands=process.NANOAODEventContent.outputCommands+[ + "drop edmTriggerResults_*_*_*" + ], + SelectEvents=cms.untracked.PSet( + SelectEvents=cms.vstring("nanoCTPath") + ) + ) +process.end = cms.EndPath(process.out) + +process.schedule = cms.Schedule(process.nanoCTPath, process.end) From 64d46b10ebb89fdb95dba82c6dac2a9540960ef5 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Mon, 23 Aug 2021 14:49:24 +0200 Subject: [PATCH 2/8] Use NanoAODOutputModule for ALCARECO stream if dataTier equals NANOAOD --- .../Applications/python/ConfigBuilder.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Configuration/Applications/python/ConfigBuilder.py b/Configuration/Applications/python/ConfigBuilder.py index f87fe67ec4ce7..a45932b9f99e7 100644 --- a/Configuration/Applications/python/ConfigBuilder.py +++ b/Configuration/Applications/python/ConfigBuilder.py @@ -1151,9 +1151,9 @@ def inGeometryKeys(opt): self.REDIGIDefaultSeq=self.DIGIDefaultSeq # for alca, skims, etc - def addExtraStream(self,name,stream,workflow='full'): + def addExtraStream(self, name, stream, workflow='full', cppType="PoolOutputModule"): # define output module and go from there - output = cms.OutputModule("PoolOutputModule") + output = cms.OutputModule(cppType) if stream.selectEvents.parameters_().__len__()!=0: output.SelectEvents = stream.selectEvents else: @@ -1187,8 +1187,9 @@ def doNotInlineEventContent(instance,label = "process."+stream.content+".outputC if self._options.filtername: output.dataset.filterName= cms.untracked.string(self._options.filtername+"_"+stream.name) - #add an automatic flushing to limit memory consumption - output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) + if cppType == "PoolOutputModule": + #add an automatic flushing to limit memory consumption + output.eventAutoFlushCompressedSize=cms.untracked.int32(5*1024*1024) if workflow in ("producers,full"): if isinstance(stream.paths,tuple): @@ -1287,10 +1288,12 @@ def prepare_ALCA(self, sequence = None, workflow = 'full'): print("Setting numberOfConcurrentLuminosityBlocks=1 because of AlCa sequence {}".format(shortName)) self._options.nConcurrentLumis = "1" self._options.nConcurrentIOVs = "1" - output = self.addExtraStream(name,alcastream, workflow = workflow) + isNano = (alcastream.dataTier == "NANOAOD") + output = self.addExtraStream(name, alcastream, workflow=workflow, + cppType=("NanoAODOutputModule" if isNano else "PoolOutputModule")) self.executeAndRemember('process.ALCARECOEventContent.outputCommands.extend(process.OutALCARECO'+shortName+'_noDrop.outputCommands)') self.AlCaPaths.append(shortName) - if 'DQM' in alcaList: + if 'DQM' in alcaList and not isNano: if not self._options.inlineEventContent and hasattr(self.process,name): self.executeAndRemember('process.' + name + '.outputCommands.append("keep *_MEtoEDMConverter_*_*")') else: From 79496e5bc432d4aae59011bd241ea02c72155d1a Mon Sep 17 00:00:00 2001 From: Pieter David Date: Mon, 23 Aug 2021 14:48:24 +0200 Subject: [PATCH 3/8] Add SiStripCalCosmicsNano definition --- ...LCARECOSiStripCalCosmicsNano_Output_cff.py | 17 +++++ .../ALCARECOSiStripCalCosmicsNano_cff.py | 65 +++++++++++++++++++ .../EventContent/python/AlCaRecoOutput_cff.py | 1 + .../python/AlCaRecoStreams_cff.py | 11 ++++ 4 files changed, 94 insertions(+) create mode 100644 Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_Output_cff.py create mode 100644 Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py diff --git a/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_Output_cff.py b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_Output_cff.py new file mode 100644 index 0000000000000..fb9691c6feb08 --- /dev/null +++ b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_Output_cff.py @@ -0,0 +1,17 @@ +import FWCore.ParameterSet.Config as cms + +# AlCaNano for track based calibration using Cosmics events +from PhysicsTools.NanoAOD.NanoAODEDMEventContent_cff import NANOAODEventContent +OutALCARECOSiStripCalCosmicsNano_noDrop = cms.PSet( + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('pathALCARECOSiStripCalCosmicsNano') + ), + outputCommands=cms.untracked.vstring( + 'keep nanoaodFlatTable_*Table_*_*', + 'keep nanoaodMergeableCounterTable_*Table_*_*', + 'keep nanoaodUniqueString_nanoMetadata_*_*', + ) + ) +import copy +OutALCARECOSiStripCalCosmicsNano = copy.deepcopy(OutALCARECOSiStripCalCosmicsNano_noDrop) +OutALCARECOSiStripCalCosmicsNano.outputCommands.insert(0, "drop *") diff --git a/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py new file mode 100644 index 0000000000000..ee97ed06c8430 --- /dev/null +++ b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py @@ -0,0 +1,65 @@ +import FWCore.ParameterSet.Config as cms + +from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmics_cff import ALCARECOSiStripCalCosmics +from CalibTracker.SiStripCommon.prescaleEvent_cfi import prescaleEvent +from HLTrigger.HLTfilters.triggerResultsFilter_cfi import triggerResultsFilter +from CalibTracker.SiStripCommon.SiStripBFieldFilter_cfi import siStripBFieldOnFilter + +ALCARECOSiStripCalCosmicsNanoPrescale = prescaleEvent.clone(prescale=1) + +ALCARECOSiStripCalCosmicsNanoHLT = triggerResultsFilter.clone( + triggerConditions=cms.vstring("HLT_L1SingleMuCosmics_v*"), + hltResults=cms.InputTag("TriggerResults", "", "HLT"), + l1tResults=cms.InputTag(""), + throw=cms.bool(False) + ) + +from CalibTracker.Configuration.Filter_Refit_cff import CalibrationTracks, CalibrationTracksRefit, MeasurementTrackerEvent, offlineBeamSpot + +ALCARECOSiStripCalCosmicsNanoCalibTracks = CalibrationTracks.clone(src=cms.InputTag("ALCARECOSiStripCalCosmics")) +ALCARECOSiStripCalCosmicsNanoCalibTracksRefit = CalibrationTracksRefit.clone( + src=cms.InputTag("ALCARECOSiStripCalCosmicsNanoCalibTracks") + ) + +ALCARECOSiStripCalCosmicsNanoTkCalSeq = cms.Sequence( + ALCARECOSiStripCalCosmicsNanoPrescale* + ALCARECOSiStripCalCosmicsNanoHLT* + siStripBFieldOnFilter* + ALCARECOSiStripCalCosmicsNanoCalibTracks, + cms.Task(MeasurementTrackerEvent), + cms.Task(offlineBeamSpot), + cms.Task(ALCARECOSiStripCalCosmicsNanoCalibTracksRefit) + ) + +from PhysicsTools.NanoAOD.common_cff import * +from PhysicsTools.NanoAOD.nano_cff import nanoMetadata +from CalibTracker.SiStripCommon.siStripPositionCorrectionsTable_cfi import siStripPositionCorrectionsTable +from CalibTracker.SiStripCommon.siStripLorentzAngleRunInfoTable_cfi import siStripLorentzAngleRunInfoTable + +ALCARECOSiStripCalCosmicsNanoTracksTable = cms.EDProducer("SimpleTrackFlatTableProducer", + src=cms.InputTag("ALCARECOSiStripCalCosmicsNanoCalibTracksRefit"), + cut=cms.string(""), + name=cms.string("track"), + doc=cms.string("SiStripCalCosmics ALCARECO tracks"), + singleton=cms.bool(False), + extension=cms.bool(False), + variables=cms.PSet( + chi2ndof=Var("chi2()/ndof", float), + pt=Var("pt()", float), + hitsvalid=Var("numberOfValidHits()", int), ## unsigned? + phi=Var("phi()", float), + eta=Var("eta()", float), + ) + ) + +ALCARECOSiStripCalCosmicsNanoMeasTable = siStripPositionCorrectionsTable.clone( + Tracks=cms.InputTag("ALCARECOSiStripCalCosmicsNanoCalibTracksRefit")) + +ALCARECOSiStripCalCosmicsNanoTables = cms.Task( + nanoMetadata, + ALCARECOSiStripCalCosmicsNanoTracksTable, + ALCARECOSiStripCalCosmicsNanoMeasTable, + siStripLorentzAngleRunInfoTable + ) + +seqALCARECOSiStripCalCosmicsNano = cms.Sequence(ALCARECOSiStripCalCosmicsNanoTkCalSeq, ALCARECOSiStripCalCosmicsNanoTables) diff --git a/Configuration/EventContent/python/AlCaRecoOutput_cff.py b/Configuration/EventContent/python/AlCaRecoOutput_cff.py index a8ddcb707fd92..ce7e83798e11e 100644 --- a/Configuration/EventContent/python/AlCaRecoOutput_cff.py +++ b/Configuration/EventContent/python/AlCaRecoOutput_cff.py @@ -53,6 +53,7 @@ from CalibTracker.SiPixelQuality.ALCARECOSiPixelCalZeroBias_Output_cff import * # AlCaReco for tracker calibration using Cosmics events from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmics_Output_cff import * +from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmicsNano_Output_cff import * from Calibration.TkAlCaRecoProducers.ALCARECOSiPixelCalCosmics_Output_cff import * # AlCaReco for tracker based alignment using beam halo diff --git a/Configuration/StandardSequences/python/AlCaRecoStreams_cff.py b/Configuration/StandardSequences/python/AlCaRecoStreams_cff.py index a5487c282e9ed..84ebf7ff9a000 100644 --- a/Configuration/StandardSequences/python/AlCaRecoStreams_cff.py +++ b/Configuration/StandardSequences/python/AlCaRecoStreams_cff.py @@ -51,6 +51,7 @@ from CalibTracker.SiPixelQuality.ALCARECOSiPixelCalZeroBias_cff import * # AlCaReco for tracker calibration using Cosmics events from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmics_cff import * +from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmicsNano_cff import * ############################################################### # LUMI Calibration @@ -184,6 +185,7 @@ pathALCARECOSiPixelCalCosmics = cms.Path(seqALCARECOSiPixelCalCosmics) pathALCARECOSiStripCalMinBias = cms.Path(seqALCARECOSiStripCalMinBias*ALCARECOSiStripCalMinBiasDQM) pathALCARECOSiStripCalCosmics = cms.Path(seqALCARECOSiStripCalCosmics) +pathALCARECOSiStripCalCosmicsNano = cms.Path(seqALCARECOSiStripCalCosmics*seqALCARECOSiStripCalCosmicsNano) pathALCARECOSiStripCalMinBiasAAG = cms.Path(seqALCARECOSiStripCalMinBiasAAG*ALCARECOSiStripCalMinBiasAAGDQM) pathALCARECOSiStripCalSmallBiasScan = cms.Path(seqALCARECOSiStripCalSmallBiasScan) pathALCARECOSiStripCalZeroBias = cms.Path(seqALCARECOSiStripCalZeroBias*ALCARECOSiStripCalZeroBiasDQM) @@ -443,6 +445,15 @@ dataTier = cms.untracked.string('ALCARECO') ) +ALCARECOStreamSiStripCalCosmicsNano = cms.FilteredStream( + responsible = "Pieter David", + name = "SiStripCalCosmicsNano", + paths = (pathALCARECOSiStripCalCosmicsNano), + content = OutALCARECOSiStripCalCosmicsNano.outputCommands, + selectEvents = OutALCARECOSiStripCalCosmicsNano.SelectEvents, + dataTier = cms.untracked.string("NANOAOD") + ) + ALCARECOStreamSiStripCalZeroBias = cms.FilteredStream( responsible = 'Gordon Kaussen', name = 'SiStripCalZeroBias', From 05134f007901fa707a8582148b2073ffb6134926 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Mon, 23 Aug 2021 15:06:24 +0200 Subject: [PATCH 4/8] Add SiStripCalCosmicsNano to relval steps --- .../PyReleaseValidation/python/relval_steps.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Configuration/PyReleaseValidation/python/relval_steps.py b/Configuration/PyReleaseValidation/python/relval_steps.py index 2d47b7e5df5ff..f4e71afd8fc11 100644 --- a/Configuration/PyReleaseValidation/python/relval_steps.py +++ b/Configuration/PyReleaseValidation/python/relval_steps.py @@ -2597,11 +2597,11 @@ def gen2021HiMix(fragment,howMuch): '-s':'ALCA:TkAlCosmics0T+MuAlGlobalCosmics+HcalCalHOCosmics+DQM' } -steps['ALCACOSDRUN2']=merge([{'--conditions':'auto:run2_data','--era':'Run2_2016','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) +steps['ALCACOSDRUN2']=merge([{'--conditions':'auto:run2_data','--era':'Run2_2016','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) -steps['ALCACOSDRUN3']=merge([{'--conditions':'auto:run3_data','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) -steps['ALCACOSDPROMPTRUN3']=merge([{'--conditions':'auto:run3_data_prompt','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) -steps['ALCACOSDEXPRUN3']=merge([{'--conditions':'auto:run3_data_express','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) +steps['ALCACOSDRUN3']=merge([{'--conditions':'auto:run3_data','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) +steps['ALCACOSDPROMPTRUN3']=merge([{'--conditions':'auto:run3_data_prompt','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) +steps['ALCACOSDEXPRUN3']=merge([{'--conditions':'auto:run3_data_express','--era':'Run3','-s':'ALCA:SiPixelCalCosmics+TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+MuAlGlobalCosmics+HcalCalHOCosmics+DQM'},steps['ALCACOSD']]) steps['ALCAPROMPT']={'-s':'ALCA:PromptCalibProd', '--filein':'file:TkAlMinBias.root', @@ -2658,10 +2658,10 @@ def gen2021HiMix(fragment,howMuch): steps['ALCAHAL']=merge([{'-s':'ALCA:TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo'},step4Up2015Defaults]) steps['ALCACOS_UP15']=merge([{'--conditions':'auto:run2_mc_cosmics','-s':'ALCA:TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo'},step4Up2015Defaults]) -steps['ALCACOS_UP16']=merge([{'--conditions':'auto:run2_mc_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2016'},step4Up2015Defaults]) -steps['ALCACOS_UP17']=merge([{'--conditions':'auto:phase1_2017_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2017'},step4Up2015Defaults]) -steps['ALCACOS_UP18']=merge([{'--conditions':'auto:phase1_2018_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2018'},step4Up2015Defaults]) -steps['ALCACOS_UP21']=merge([{'--conditions':'auto:phase1_2021_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run3'},step4Up2015Defaults]) +steps['ALCACOS_UP16']=merge([{'--conditions':'auto:run2_mc_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2016'},step4Up2015Defaults]) +steps['ALCACOS_UP17']=merge([{'--conditions':'auto:phase1_2017_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2017'},step4Up2015Defaults]) +steps['ALCACOS_UP18']=merge([{'--conditions':'auto:phase1_2018_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run2_2018'},step4Up2015Defaults]) +steps['ALCACOS_UP21']=merge([{'--conditions':'auto:phase1_2021_cosmics','-s':'ALCA:TkAlCosmics0T+SiStripCalCosmics+SiStripCalCosmicsNano+SiPixelCalCosmics+TkAlBeamHalo+MuAlBeamHaloOverlaps+MuAlBeamHalo','--era':'Run3'},step4Up2015Defaults]) steps['ALCACOS_UP21_0T']=merge([{'--magField':'0T','--conditions':'auto:phase1_2021_cosmics_0T'},steps['ALCACOS_UP21']]) steps['ALCAHARVD']={'-s':'ALCAHARVEST:BeamSpotByRun+BeamSpotByLumi+SiStripQuality', '--conditions':'auto:run1_data', From e81a5d7a521c86a8805602ef027527e492c1a4fc Mon Sep 17 00:00:00 2001 From: Pieter David Date: Mon, 23 Aug 2021 17:19:55 +0200 Subject: [PATCH 5/8] Remove B-field filter --- .../python/ALCARECOSiStripCalCosmicsNano_cff.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py index ee97ed06c8430..59cb4c8ac3789 100644 --- a/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py +++ b/Calibration/TkAlCaRecoProducers/python/ALCARECOSiStripCalCosmicsNano_cff.py @@ -3,7 +3,6 @@ from Calibration.TkAlCaRecoProducers.ALCARECOSiStripCalCosmics_cff import ALCARECOSiStripCalCosmics from CalibTracker.SiStripCommon.prescaleEvent_cfi import prescaleEvent from HLTrigger.HLTfilters.triggerResultsFilter_cfi import triggerResultsFilter -from CalibTracker.SiStripCommon.SiStripBFieldFilter_cfi import siStripBFieldOnFilter ALCARECOSiStripCalCosmicsNanoPrescale = prescaleEvent.clone(prescale=1) @@ -24,7 +23,6 @@ ALCARECOSiStripCalCosmicsNanoTkCalSeq = cms.Sequence( ALCARECOSiStripCalCosmicsNanoPrescale* ALCARECOSiStripCalCosmicsNanoHLT* - siStripBFieldOnFilter* ALCARECOSiStripCalCosmicsNanoCalibTracks, cms.Task(MeasurementTrackerEvent), cms.Task(offlineBeamSpot), From 0f0c650fff698edb0367e7a1bfb4a6e2f2e167a7 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Wed, 25 Aug 2021 11:15:26 +0200 Subject: [PATCH 6/8] Add SiStripCalCosmicsNano tests for Configuration/DataProcessing --- Configuration/DataProcessing/test/run_CfgTest.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Configuration/DataProcessing/test/run_CfgTest.sh b/Configuration/DataProcessing/test/run_CfgTest.sh index ad04c1d7c2db6..9d09dbb9e99a3 100755 --- a/Configuration/DataProcessing/test/run_CfgTest.sh +++ b/Configuration/DataProcessing/test/run_CfgTest.sh @@ -21,6 +21,12 @@ do runTest "${LOCAL_TEST_DIR}/RunAlcaHarvesting.py --scenario $scenario --lfn /store/whatever --dataset /A/B/C --global-tag GLOBALTAG --workflows=BeamSpotByRun,BeamSpotByLumi,SiStripQuality" done +declare -a arr=("cosmicsEra_Run2_2018" "cosmicsHybridEra_Run2_2018" "cosmicsEra_Run3") +for scenario in "${arr[@]}" +do + runTest "${LOCAL_TEST_DIR}/RunExpressProcessing.py --scenario $scenario --global-tag GLOBALTAG --lfn /store/whatever --fevt --dqmio --alcareco SiStripCalCosmicsNano " +done + declare -a arr=("HeavyIonsEra_Run2_2018") for scenario in "${arr[@]}" do From bc66b49f7f90b283da46b29c072e520bb7d86024 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Wed, 1 Sep 2021 10:06:19 +0200 Subject: [PATCH 7/8] Remove a comment --- .../produceCalibrationTree_CosmicsLABP_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py b/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py index 6341d7b945525..bc5062a7a833b 100644 --- a/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py +++ b/CalibTracker/SiStripCommon/test/MakeCalibrationTrees/produceCalibrationTree_CosmicsLABP_cfg.py @@ -75,7 +75,7 @@ process.source = cms.Source("PoolSource", fileNames=cms.untracked.vstring(options.inputFiles)) if options.runNumber != -1: - if 'Cosmics' not in options.inputCollection: # FIXME: this should be improved + if 'Cosmics' not in options.inputCollection: print("Restricting to the following events :") print('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) process.source.eventsToProcess = cms.untracked.VEventRange('%s:1-%s:MAX'%(options.runNumber,options.runNumber)) From c52defb96dd625233f218921184017b1fed4e195 Mon Sep 17 00:00:00 2001 From: Pieter David Date: Wed, 1 Sep 2021 10:31:53 +0200 Subject: [PATCH 8/8] update inspectNanoFile script for Runs and LuminosityBlocks trees They are only added to the outputs if they contain non-standard products (i.e. more branches than the run and lumiblock number) --- PhysicsTools/NanoAOD/test/inspectNanoFile.py | 337 +++++++++++-------- 1 file changed, 199 insertions(+), 138 deletions(-) diff --git a/PhysicsTools/NanoAOD/test/inspectNanoFile.py b/PhysicsTools/NanoAOD/test/inspectNanoFile.py index 731d1f4254ddd..d05154a5aab72 100755 --- a/PhysicsTools/NanoAOD/test/inspectNanoFile.py +++ b/PhysicsTools/NanoAOD/test/inspectNanoFile.py @@ -15,6 +15,10 @@ def __init__(self,data): setattr(self,k,v) self.Events = self.trees["Events"] self.nevents = self.Events["entries"] + self.Runs = self.trees["Runs"] + self.nruns = self.Runs["entries"] + self.LuminosityBlocks = self.trees["LuminosityBlocks"] + self.nluminosityblocks = self.LuminosityBlocks["entries"] class Branch: def __init__(self, tree, branch): @@ -87,13 +91,13 @@ def inspectRootFile(infile): filesize = os.path.getsize(infile)/1024.0 tfile = ROOT.TFile.Open(infile) trees = {} - for treeName in "Events", "Runs", "Lumis": + for treeName in "Events", "Runs", "LuminosityBlocks": toplevelDoc={} tree = tfile.Get(treeName) entries = tree.GetEntries() trees[treeName] = tree branchList = tree.GetListOfBranches() - allbranches = [ Branch(tree,branchList.At(i)) for i in range(branchList.GetSize()) ] + allbranches = [ Branch(tree, br) for br in branchList ] branchmap = dict((b.name,b) for b in allbranches) branchgroups = {} # make list of counters and countees @@ -149,7 +153,6 @@ def inspectRootFile(infile): branchgroups = dict(bg.toJSON() for bg in branchgroups.values()), ) c1.Close() - break # only Event tree for now tfile.Close() return dict(filename = os.path.basename(infile), filesize = filesize, trees = trees) @@ -175,11 +178,13 @@ def makeSurvey(treeName, treeData): runningtotal += s['tot'] return (survey, "\n,\t".join(scriptdata)) -def writeSizeReport(fileData, stream): +def writeSizeReport(fileData, trees, stream): filename = fileData.filename filesize = fileData.filesize events = fileData.nevents - survey, scriptdata = makeSurvey("Events", filedata.Events) + surveys = {} + for treename, treeData in trees.items(): + surveys[treename] = makeSurvey(treename, treeData) title = "%s (%.3f Mb, %d events, %.2f kb/event)" % (filename, filesize/1024.0, events, filesize/events) stream.write(""" @@ -194,87 +199,115 @@ def writeSizeReport(fileData, stream):

{title}

 - [No canvas support] - -

Event data

- - """); + """.format( + "\n".join(""" + values = []; + labels = []; + keys = []; + tips = []; + for (var i = 0; i < data_{treename}.length; i++) {{ + values.push( data_{treename}[i].size ); + labels.push( data_{treename}[i].label ); + keys.push( data_{treename}[i].label ); + tips.push( data_{treename}[i].tip ); + }} + var chart_{treename} = new RGraph.Pie("{treename}Canvas", values) + .Set('exploded', 7) + .Set('tooltips', tips) + .Set('tooltips.event', 'onmousemove') + .Set('key', labels) + .Set('key.position.graph.boxed', false) + .Draw(); + """.format(treename=treename, scriptdata=scriptdata) + for treename, (_, scriptdata) in surveys.items()))) + + if len(trees) > 1: + stream.write("

Collections summary

\n") + stream.write("
\n") stream.write("\n"); - grandtotal = filedata.Events['allsize']; runningtotal = 0 - for s in survey: - stream.write("" % (s['doc'],s['name'],s['name'],s['kind'].lower(),len(s['subs']))) - stream.write("" % (s['entries']/events, s['tot']/events, s['tot']/s['entries']*1024 if s['entries'] else 0)) - stream.write("" % (s['tot']/grandtotal*200,10)) - stream.write("" % ( s['tot']/grandtotal * 100.0)) - stream.write("" % ( (runningtotal+s['tot'])/grandtotal * 100.0)) - stream.write("" % ( (grandtotal-runningtotal)/grandtotal * 100.0)) + runningtotal = 0 + for treename, (survey, _) in surveys.items(): + treetotal = trees[treename]['allsize'] + treerunningtotal = 0 + for s in survey: + stream.write("" % (s['doc'],s['name'],s['name'],s['kind'].lower(),len(s['subs']))) + stream.write("" % (s['entries']/events, s['tot']/events, s['tot']/s['entries']*1024 if s['entries'] else 0)) + stream.write("" % (s['tot']/treetotal*200,10)) + stream.write("" % ( s['tot']/treetotal * 100.0)) + stream.write("" % ( (runningtotal+s['tot'])/treetotal * 100.0)) + stream.write("" % ( (treetotal-runningtotal)/treetotal * 100.0)) + stream.write("\n") + treerunningtotal += s['tot'] + runningtotal += treerunningtotal + + # all known data + stream.write("" % treename) + stream.write("" % (treetotal/events)) + stream.write("" % (treetotal/filesize*100.0)) stream.write("\n") - runningtotal += s['tot']; - # all known data - stream.write("") - stream.write("" % (grandtotal/events)) - stream.write("" % (grandtotal/filesize*100.0)) - stream.write("\n") + if treename == "Events": + # non-event + stream.write("") + stream.write("" % ( (filesize-treetotal)/events)) + stream.write("" % ( (filesize-treetotal)/filesize * 100, 10 )) + stream.write("" % ( (filesize-treetotal)/filesize * 100.0 )) + stream.write("\n") - # other, unknown overhead - stream.write("") - stream.write("" % ( (filesize-grandtotal)/events)) - stream.write("" % ( (filesize-grandtotal)/filesize * 100, 10 )) - stream.write("" % ( (filesize-grandtotal)/filesize * 100.0 )) - stream.write("\n") + if len(surveys) > 1: + # other, unknown overhead + stream.write("") + stream.write("" % ( (filesize-runningtotal)/events)) + stream.write("" % ( (filesize-runningtotal)/filesize * 100, 10 )) + stream.write("" % ( (filesize-runningtotal)/filesize * 100.0 )) + stream.write("\n") # all file - stream.write("") + stream.write("") stream.write("" % (filesize/events)) stream.write("\n") stream.write("""
" + "".join([ "collection", "kind", "vars", "items/evt", "kb/evt", "b/item", "plot", "%" ]) + "cumulative %
%s%s%d%.2f%.3f%.1f%.1f%%%.1f%%%.1f%%
%s%s%d%.2f%.3f%.1f%.1f%%%.1f%%%.1f%%
All %s data   %.2f " % ( treetotal/filesize*100.0)) + stream.write("%.1f%%a
All Event data   %.2f " % ( grandtotal/filesize*100.0)) - stream.write("%.1f%%a
Non per-event data or overhead   %.2f %.1f%%a
Non per-event data or overhead   %.2f %.1f%%a
Overhead   %.2f %.1f%%a
File size
File size   %.2f   
- Note: size percentages of individual event products are relative to the total size of Event data only.
+ Note: size percentages of individual event products are relative to the total size of Event data only (or equivalent for non-Events trees).
Percentages with a are instead relative to the full file size. -

Events detail

""") - for s in sorted(survey, key = lambda s : s['name']): - stream.write("

%s (%.1f items/evt, %.3f kb/evt) [back to top]

" % (s['name'], s['name'], s['name'], s['entries']/events, s['tot']/events)) - stream.write("\n") - stream.write("\n") - subs = [ fileData.Events['branches'][b] for b in s['subs'] ] - for b in sorted(subs, key = lambda s : - s['tot']): - stream.write("" % (b['doc'],b['name'], b['kind'], b['tot']/events*1024, b['tot']/s['entries']*1024 if s['entries'] else 0)) - stream.write("" % ( b['tot']/s['tot']*200, 10 )) - stream.write("" % (b['tot']/s['tot'] * 100.0)) - stream.write("\n") - stream.write("
" + "".join( [ "branch", "kind", "b/event", "b/item", "plot", "%" ]) + "
%s%s%.1f%.1f%.1f%%
\n") + for treename, treeData in trees.items(): + stream.write(""" +

%s detail

+ """ % treename) + for s in sorted(surveys[treename][0], key = lambda s : s['name']): + stream.write("

%s (%.1f items/evt, %.3f kb/evt) [back to top]

" % (s['name'], s['name'], s['name'], s['entries']/events, s['tot']/events)) + stream.write("\n") + stream.write("\n") + subs = [ treeData['branches'][b] for b in s['subs'] ] + for b in sorted(subs, key = lambda s : - s['tot']): + stream.write("" % (b['doc'],b['name'], b['kind'], b['tot']/events*1024, b['tot']/s['entries']*1024 if s['entries'] else 0)) + stream.write("" % ( b['tot']/s['tot']*200, 10 )) + stream.write("" % (b['tot']/s['tot'] * 100.0)) + stream.write("\n") + stream.write("
" + "".join( [ "branch", "kind", "b/event", "b/item", "plot", "%" ]) + "
%s%s%.1f%.1f%.1f%%
\n") stream.write(""" """) -def writeDocReport(fileData, stream): +def writeDocReport(fileName, trees, stream): stream.write( """ @@ -282,95 +315,115 @@ def writeDocReport(fileData, stream): -

Content

- - """.format(filename=fileData.filename)) - stream.write( "\n" ) - groups = list(fileData.Events['branchgroups'].values()) - groups.sort(key = lambda s : s['name']) - for s in groups: - stream.write( "\n" % (s['name'],s['name'],s['doc']) ) - stream.write( "
CollectionDescription
%s%s
\n\n

Events detail

\n" ) - for s in groups: - stream.write( "

%s [back to top]

\n" % (s['name'], s['name'], s['name']) ) - stream.write( "\n" ) - stream.write( "\n" ) - subs = [ fileData.Events['branches'][b] for b in s['subs'] ] - for b in sorted(subs, key = lambda s : s['name']): - stream.write( "" % (b['name'], b['kind'], b['doc']) ) - stream.write( "\n" ) - stream.write( "
Object propertyTypeDescription
%s%s%s
\n" ) + """.format(filename=fileName)) + for treename, treeData in trees.items(): + stream.write(""" +

{treename} Content

+ + """.format(treename=treename)) + stream.write( "\n" ) + groups = list(treeData['branchgroups'].values()) + groups.sort(key = lambda s : s['name']) + for s in groups: + stream.write( "\n" % (s['name'],s['name'],s['doc']) ) + stream.write("
CollectionDescription
%s%s
\n\n

{treename} detail

\n".format(treename=treename)) + for s in groups: + stream.write( "

%s [back to top]

\n" % (s['name'], s['name'], s['name']) ) + stream.write( "\n" ) + stream.write( "\n" ) + subs = [ treeData['branches'][b] for b in s['subs'] ] + for b in sorted(subs, key = lambda s : s['name']): + stream.write( "" % (b['name'], b['kind'], b['doc']) ) + stream.write( "\n" ) + stream.write( "
Object propertyTypeDescription
%s%s%s
\n" ) stream.write( """ """ ) -def writeMarkdownSizeReport(fileData, stream): +def writeMarkdownSizeReport(fileData, trees, stream): filename = fileData.filename filesize = fileData.filesize events = fileData.nevents - survey, _ = makeSurvey("Events", filedata.Events) + surveys = {} + for treename, treeData in trees.items(): + surveys[treename] = makeSurvey(treename, treeData)[0] stream.write("**%s (%.3f Mb, %d events, %.2f kb/event)**\n" % (filename, filesize/1024.0, events, filesize/events)) - stream.write("\n# Event data\n") + stream.write("\n# Event data\n" if len(trees) == 1 else "\n# Collection data\n") stream.write("| collection | kind | vars | items/evt | kb/evt | b/item | plot | % | ascending cumulative % | descending cumulative % |\n") stream.write("| - | - | - | - | - | - | - | - | - | - |\n") - grandtotal = filedata.Events['allsize']; runningtotal = 0 - for s in survey: - stream.write("| [**%s**](#%s '%s') | %s | %d" % (s['name'], s['name'].lower(), s['doc'].replace('|', '\|').replace('\'', '\"'), s['kind'].lower(), len(s['subs']))) - stream.write("| %.2f|%.3f|%.1f" % (s['entries']/events, s['tot']/events, s['tot'] / s['entries'] * 1024 if s['entries'] else 0)) - stream.write("| " % (s['tot'] / grandtotal * 200, 10)) - stream.write("| %.1f%%" % (s['tot'] / grandtotal * 100.0)) - stream.write("| %.1f%%" % ((runningtotal+s['tot'])/grandtotal * 100.0)) - stream.write("| %.1f%% |\n" % ((grandtotal-runningtotal)/grandtotal * 100.0)) - runningtotal += s['tot'] + runningtotal = 0 + for treename, survey in surveys.items(): + treetotal = trees[treename]['allsize'] + treerunningtotal = 0 + for s in survey: + stream.write("| [**%s**](#%s '%s') | %s | %d" % (s['name'], s['name'].lower(), s['doc'].replace('|', '\|').replace('\'', '\"'), s['kind'].lower(), len(s['subs']))) + stream.write("| %.2f|%.3f|%.1f" % (s['entries']/events, s['tot']/events, s['tot'] / s['entries'] * 1024 if s['entries'] else 0)) + stream.write("| " % (s['tot'] / treetotal * 200, 10)) + stream.write("| %.1f%%" % (s['tot'] / treetotal * 100.0)) + stream.write("| %.1f%%" % ((runningtotal+s['tot'])/treetotal * 100.0)) + stream.write("| %.1f%% |\n" % ((treetotal-runningtotal)/treetotal * 100.0)) + runningtotal += s['tot'] - # all known data - stream.write("**All Event data**") - stream.write("| | | | **%.2f**" % (grandtotal/events)) - stream.write("| | " % (grandtotal / filesize * 100.0, 10)) - stream.write("| %.1f%%a | | |\n" % (grandtotal/filesize * 100.0)) + # all known data + stream.write("**All %s data**" % treename) + stream.write("| | | | **%.2f**" % (treetotal/events)) + stream.write("| | " % (treetotal / filesize * 100.0, 10)) + stream.write("| %.1f%%a | | |\n" % (treetotal/filesize * 100.0)) - # other, unknown overhead - stream.write("**Non per-event data or overhead**") - stream.write("| | | | %.2f" % ((filesize-grandtotal)/events)) - stream.write("| | " % ((filesize - grandtotal) / filesize * 100, 10)) - stream.write("| %.1f%%a | | |\n" % ((filesize-grandtotal)/filesize * 100.0)) + if treename == "Events": + # non-event + stream.write("**Non per-event data or overhead**") + stream.write("| | | | %.2f" % ((filesize-treetotal)/events)) + stream.write("| | " % ((filesize - treetotal) / filesize * 100, 10)) + stream.write("| %.1f%%a | | |\n" % ((filesize-treetotal)/filesize * 100.0)) + + if len(surveys) > 1: + # other, unknown overhead + stream.write("**Overhead**") + stream.write("| | | | %.2f" % ((filesize-runningtotal)/events)) + stream.write("| | " % ((filesize - runningtotal) / filesize * 100, 10)) + stream.write("| %.1f%%a | | |\n" % ((filesize-runningtotal)/filesize * 100.0)) # all file stream.write("**File size**") stream.write("| | | | **%.2f**" % (filesize/events)) stream.write("| | | | | |\n\n") - stream.write("Note: size percentages of individual event products are relative to the total size of Event data only.\\\n") + stream.write("Note: size percentages of individual event products are relative to the total size of Event data only (or equivalent for non-Events trees).\\\n") stream.write("Percentages with a are instead relative to the full file size.\n\n") - stream.write("# Events detail\n") - for s in sorted(survey, key=lambda s: s['name']): - stream.write("## %s (%.1f items/evt, %.3f kb/evt) [[back to top]](#event-data)\n" % (s['name'].lower(), s['name'], s['entries'] / events, s['tot'] / events)) - stream.write("| branch | kind | b/event | b/item | plot | % |\n") - stream.write("| - | - | - | - | - | - |\n") - subs = [fileData.Events['branches'][b] for b in s['subs']] - for b in sorted(subs, key = lambda s: - s['tot']): - stream.write("| %s | %s | %.1f | %.1f" % (b['doc'].replace('|', '\|').replace('\'', '\"'), b['name'], b['kind'], b['tot'] / events * 1024, b['tot'] / s['entries'] * 1024 if s['entries'] else 0)) - stream.write("| " % (b['tot'] / s['tot'] * 200, 10)) - stream.write("| %.1f%% |\n" % (b['tot'] / s['tot'] * 100.0)) + for treename, survey in surveys.items(): + stream.write("# %s detail\n" % treename) + for s in sorted(survey, key=lambda s: s['name']): + stream.write("## %s (%.1f items/evt, %.3f kb/evt) [[back to top]](#event-data)\n" % (s['name'].lower(), s['name'], s['entries'] / events, s['tot'] / events)) + stream.write("| branch | kind | b/event | b/item | plot | % |\n") + stream.write("| - | - | - | - | - | - |\n") + subs = [trees[treename]['branches'][b] for b in s['subs']] + for b in sorted(subs, key = lambda s: - s['tot']): + stream.write("| %s | %s | %.1f | %.1f" % (b['doc'].replace('|', '\|').replace('\'', '\"'), b['name'], b['kind'], b['tot'] / events * 1024, b['tot'] / s['entries'] * 1024 if s['entries'] else 0)) + stream.write("| " % (b['tot'] / s['tot'] * 200, 10)) + stream.write("| %.1f%% |\n" % (b['tot'] / s['tot'] * 100.0)) + stream.write("\n") -def writeMarkdownDocReport(fileData, stream): - stream.write("# Content\n") - stream.write("\n| Collection | Description |\n") - stream.write("| - | - |\n") - groups = fileData.Events['branchgroups'].values() - groups.sort(key = lambda s : s['name']) - for s in groups: - stream.write("| [**%s**](#%s) | %s |\n" % (s['name'], s['name'].lower(), s['doc'])) - stream.write("\n# Events detail\n") - for s in groups: - stream.write("\n## %s [[back to top]](#content)\n" % (s['name'].lower(), s['name'])) - stream.write("| Object property | Type | Description |\n") - stream.write("| - | - | - |\n") - subs = [fileData.Events['branches'][b] for b in s['subs']] - for b in sorted(subs, key = lambda s : s['name']): - stream.write("| **%s** | %s| %s |\n" % (b['name'], b['kind'], b['doc'])) +def writeMarkdownDocReport(trees, stream): + for treename, treeData in trees.items(): + stream.write("# %s Content\n" % treename) + stream.write("\n| Collection | Description |\n") + stream.write("| - | - |\n") + groups = list(treeData['branchgroups'].values()) + groups.sort(key = lambda s : s['name']) + for s in groups: + stream.write("| [**%s**](#%s) | %s |\n" % (s['name'], s['name'].lower(), s['doc'])) + stream.write("\n# %s detail\n" % treename) + for s in groups: + stream.write("\n## %s [[back to top]](#content)\n" % (s['name'].lower(), s['name'])) + stream.write("| Object property | Type | Description |\n") + stream.write("| - | - | - |\n") + subs = [treeData['branches'][b] for b in s['subs']] + for b in sorted(subs, key = lambda s : s['name']): + stream.write("| **%s** | %s| %s |\n" % (b['name'], b['kind'], b['doc'])) + stream.write("\n") def _maybeOpen(filename): return open(filename, 'w') if filename != "-" else sys.stdout @@ -395,15 +448,23 @@ def _maybeOpen(filename): if options.json: json.dump(filedata._json, _maybeOpen(options.json), indent=4) sys.stderr.write("JSON output saved to %s\n" % options.json) + + treedata = {} # trees for (HTML or markdown) doc report + if len(filedata.Runs["branches"]) > 1: # default: run number + treedata["Runs"] = filedata.Runs + if len(filedata.LuminosityBlocks["branches"]) > 2: # default: run number, lumiblock + treedata["LuminosityBlocks"] = filedata.LuminosityBlocks + treedata["Events"] = filedata.Events + if options.doc: - writeDocReport(filedata, _maybeOpen(options.doc)) + writeDocReport(filedata.filename, treedata, _maybeOpen(options.doc)) sys.stderr.write("HTML documentation saved to %s\n" % options.doc) if options.size: - writeSizeReport(filedata, _maybeOpen(options.size)) + writeSizeReport(filedata, treedata, _maybeOpen(options.size)) sys.stderr.write("HTML size report saved to %s\n" % options.size) if options.docmd: - writeMarkdownDocReport(filedata, _maybeOpen(options.docmd)) + writeMarkdownDocReport(treedata, _maybeOpen(options.docmd)) sys.stderr.write("Markdown documentation saved to %s\n" % options.docmd) if options.sizemd: - writeMarkdownSizeReport(filedata, _maybeOpen(options.sizemd)) + writeMarkdownSizeReport(filedata, treedata, _maybeOpen(options.sizemd)) sys.stderr.write("Markdown size report saved to %s\n" % options.sizemd)