From b0972e17945cda72ed1b1b30848f438a44161da2 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Tue, 15 Sep 2020 23:02:40 +0200 Subject: [PATCH 01/26] Implement persistence for ProcessBlock products --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 6 +- DataFormats/Provenance/interface/BranchType.h | 5 + .../interface/EventToProcessBlockIndexes.h | 24 + .../Provenance/interface/ProvenanceFwd.h | 2 + .../interface/StoredProcessBlockHelper.h | 50 +++ DataFormats/Provenance/src/BranchType.cc | 12 + DataFormats/Provenance/src/ProductRegistry.cc | 3 +- .../src/StoredProcessBlockHelper.cc | 10 + DataFormats/Provenance/src/classes.h | 2 + DataFormats/Provenance/src/classes_def.xml | 7 + DataFormats/Provenance/test/BuildFile.xml | 4 + .../test/StoredProcessBlockHelper_t.cpp | 46 ++ FWCore/Common/interface/FWCoreCommonFwd.h | 12 + .../interface/OutputProcessBlockHelper.h | 76 ++++ FWCore/Common/interface/ProcessBlockHelper.h | 120 +++++ .../Common/interface/ProcessBlockHelperBase.h | 67 +++ .../Common/interface/SubProcessBlockHelper.h | 35 ++ FWCore/Common/src/OutputProcessBlockHelper.cc | 231 ++++++++++ FWCore/Common/src/ProcessBlockHelper.cc | 279 ++++++++++++ FWCore/Common/src/ProcessBlockHelperBase.cc | 56 +++ FWCore/Common/src/SubProcessBlockHelper.cc | 75 ++++ .../test/test_catch2_ProcessBlockHelpers.cc | 36 ++ FWCore/Framework/interface/CacheHandle.h | 35 ++ FWCore/Framework/interface/EDAnalyzer.h | 1 + FWCore/Framework/interface/EDConsumerBase.h | 11 +- FWCore/Framework/interface/EDFilter.h | 1 + FWCore/Framework/interface/EDProducer.h | 1 + FWCore/Framework/interface/Event.h | 4 + FWCore/Framework/interface/EventForOutput.h | 2 + FWCore/Framework/interface/EventPrincipal.h | 13 +- FWCore/Framework/interface/EventProcessor.h | 6 +- FWCore/Framework/interface/FileBlock.h | 47 +- FWCore/Framework/interface/Frameworkfwd.h | 1 + .../interface/InputProcessBlockCacheImpl.h | 234 ++++++++++ FWCore/Framework/interface/InputSource.h | 73 ++-- .../interface/InputSourceDescription.h | 4 + .../Framework/interface/OccurrenceForOutput.h | 2 +- FWCore/Framework/interface/Principal.h | 2 + .../Framework/interface/PrincipalGetAdapter.h | 2 + FWCore/Framework/interface/ProcessBlock.h | 2 + .../interface/ProcessBlockForOutput.h | 41 ++ FWCore/Framework/interface/Schedule.h | 4 +- FWCore/Framework/interface/ScheduleItems.h | 10 +- FWCore/Framework/interface/WorkerManager.h | 5 +- .../interface/global/EDAnalyzerBase.h | 8 +- .../Framework/interface/global/EDFilterBase.h | 7 +- .../interface/global/EDProducerBase.h | 7 +- .../interface/global/OutputModuleBase.h | 12 +- .../global/analyzerAbilityToImplementor.h | 18 +- .../global/filterAbilityToImplementor.h | 42 +- .../Framework/interface/global/implementors.h | 59 ++- .../global/producerAbilityToImplementor.h | 44 +- .../interface/limited/EDAnalyzerBase.h | 8 +- .../interface/limited/EDFilterBase.h | 8 +- .../interface/limited/EDProducerBase.h | 6 +- .../interface/limited/OutputModuleBase.h | 13 +- .../limited/analyzerAbilityToImplementor.h | 18 +- .../limited/filterAbilityToImplementor.h | 40 +- .../interface/limited/implementors.h | 57 ++- .../limited/producerAbilityToImplementor.h | 42 +- FWCore/Framework/interface/moduleAbilities.h | 3 +- .../Framework/interface/one/EDAnalyzerBase.h | 8 +- FWCore/Framework/interface/one/EDFilterBase.h | 8 +- .../Framework/interface/one/EDProducerBase.h | 7 +- .../interface/one/OutputModuleBase.h | 12 +- .../one/analyzerAbilityToImplementor.h | 18 +- .../one/filterAbilityToImplementor.h | 30 +- FWCore/Framework/interface/one/implementors.h | 54 ++- .../one/producerAbilityToImplementor.h | 32 +- .../interface/processBlockUtilities.h | 19 + .../interface/stream/AbilityChecker.h | 4 +- .../interface/stream/AbilityToImplementor.h | 34 +- .../interface/stream/CacheContexts.h | 29 +- .../Framework/interface/stream/EDAnalyzer.h | 42 +- .../interface/stream/EDAnalyzerAdaptor.h | 16 +- .../interface/stream/EDAnalyzerAdaptorBase.h | 12 +- FWCore/Framework/interface/stream/EDFilter.h | 40 +- .../Framework/interface/stream/EDProducer.h | 44 +- .../interface/stream/ProducingModuleAdaptor.h | 18 +- .../stream/ProducingModuleAdaptorBase.h | 12 +- .../interface/stream/callAbilities.h | 89 +++- .../Framework/interface/stream/implementors.h | 97 ++++- .../Framework/interface/stream/makeGlobal.h | 12 +- FWCore/Framework/src/EDConsumerBase.cc | 2 + FWCore/Framework/src/EventForOutput.cc | 4 + FWCore/Framework/src/EventPrincipal.cc | 15 +- FWCore/Framework/src/EventProcessor.cc | 37 +- FWCore/Framework/src/FileBlock.cc | 38 ++ FWCore/Framework/src/GlobalSchedule.cc | 5 +- FWCore/Framework/src/GlobalSchedule.h | 3 +- FWCore/Framework/src/InputSource.cc | 36 +- .../Framework/src/OutputModuleCommunicator.h | 5 +- .../src/OutputModuleCommunicatorT.cc | 5 +- .../Framework/src/OutputModuleCommunicatorT.h | 5 +- FWCore/Framework/src/Principal.cc | 4 + FWCore/Framework/src/PrincipalGetAdapter.cc | 4 + FWCore/Framework/src/ProcessBlock.cc | 2 + FWCore/Framework/src/ProcessBlockForOutput.cc | 13 + FWCore/Framework/src/ProductResolvers.cc | 15 +- FWCore/Framework/src/ProductResolvers.h | 21 +- FWCore/Framework/src/Schedule.cc | 15 +- FWCore/Framework/src/ScheduleItems.cc | 11 +- FWCore/Framework/src/SubProcess.cc | 93 +++- FWCore/Framework/src/SubProcess.h | 9 +- FWCore/Framework/src/Worker.h | 5 +- FWCore/Framework/src/WorkerManager.cc | 4 +- FWCore/Framework/src/WorkerT.cc | 11 + FWCore/Framework/src/WorkerT.h | 4 + FWCore/Framework/src/global/EDAnalyzerBase.cc | 10 +- FWCore/Framework/src/global/EDFilterBase.cc | 10 +- FWCore/Framework/src/global/EDProducerBase.cc | 10 +- .../Framework/src/global/OutputModuleBase.cc | 15 +- FWCore/Framework/src/globalTransitionAsync.h | 48 +- .../Framework/src/insertSelectedProcesses.cc | 7 +- .../Framework/src/insertSelectedProcesses.h | 4 +- .../Framework/src/limited/EDAnalyzerBase.cc | 10 +- FWCore/Framework/src/limited/EDFilterBase.cc | 10 +- .../Framework/src/limited/EDProducerBase.cc | 10 +- .../Framework/src/limited/OutputModuleBase.cc | 15 +- FWCore/Framework/src/one/EDAnalyzerBase.cc | 10 +- FWCore/Framework/src/one/EDFilterBase.cc | 10 +- FWCore/Framework/src/one/EDProducerBase.cc | 10 +- FWCore/Framework/src/one/OutputModuleBase.cc | 15 +- FWCore/Framework/src/processBlockUtilities.cc | 11 + .../src/stream/EDAnalyzerAdaptorBase.cc | 3 - .../src/stream/ProducingModuleAdaptorBase.cc | 5 - .../test/stubs/TestGlobalAnalyzers.cc | 410 ++++++++++++++++- .../Framework/test/stubs/TestGlobalFilters.cc | 167 ++++++- .../test/stubs/TestGlobalProducers.cc | 171 +++++++- .../test/stubs/TestLimitedAnalyzers.cc | 169 ++++++- .../test/stubs/TestLimitedFilters.cc | 171 +++++++- .../test/stubs/TestLimitedProducers.cc | 171 +++++++- .../Framework/test/stubs/TestModuleDelete.cc | 2 +- .../Framework/test/stubs/TestOneAnalyzers.cc | 284 +++++++++++- FWCore/Framework/test/stubs/TestOneFilters.cc | 160 ++++++- .../Framework/test/stubs/TestOneProducers.cc | 157 ++++++- .../test/stubs/TestStreamAnalyzers.cc | 277 +++++++++++- .../Framework/test/stubs/TestStreamFilters.cc | 253 ++++++++++- .../test/stubs/TestStreamProducers.cc | 247 ++++++++++- .../Framework/test/stubs/ToyIntProducers.cc | 64 ++- .../test_catch2notTP_InputProcessBlock.cc | 68 +++ ...atch2notTP_InputProcessBlockCacheHolder.cc | 168 +++++++ .../test/test_module_delete_subprocess_cfg.py | 20 +- FWCore/Integration/test/BuildFile.xml | 7 +- .../Integration/test/PutOrMergeTestSource.cc | 6 +- FWCore/Integration/test/TestFindProduct.cc | 129 ++++-- FWCore/Integration/test/TestGlobalOutput.cc | 88 +++- FWCore/Integration/test/TestLimitedOutput.cc | 89 +++- FWCore/Integration/test/TestOneOutput.cc | 412 ++++++++++++++++++ FWCore/Integration/test/ThrowingSource.cc | 6 +- FWCore/Integration/test/run_TestOutput.sh | 4 + .../Integration/test/run_TestProcessBlock.sh | 182 ++++++++ .../Integration/test/testConsumesInfo_cfg.py | 21 +- FWCore/Integration/test/testGetBy1Mod_cfg.py | 8 +- FWCore/Integration/test/testGetBy3_cfg.py | 9 + .../test/testLooperEventNavigation2.txt | 24 + .../test/testLooperEventNavigation2_cfg.py | 41 ++ .../test/testLooperEventNavigation3.txt | 6 + .../test/testLooperEventNavigation3_cfg.py | 42 ++ .../test/testLooperEventNavigation_cfg.py | 5 - .../Integration/test/testProcessBlock1_cfg.py | 98 +++++ .../test/testProcessBlock2Dropped_cfg.py | 74 ++++ .../Integration/test/testProcessBlock2_cfg.py | 45 ++ .../Integration/test/testProcessBlock3_cfg.py | 36 ++ .../Integration/test/testProcessBlock4_cfg.py | 36 ++ .../Integration/test/testProcessBlock5_cfg.py | 38 ++ .../test/testProcessBlockDropOnInput_cfg.py | 82 ++++ .../test/testProcessBlockDropOnOutput2_cfg.py | 92 ++++ .../test/testProcessBlockDropOnOutput_cfg.py | 53 +++ .../test/testProcessBlockFailMerge_cfg.py | 32 ++ .../test/testProcessBlockMerge2_cfg.py | 59 +++ .../test/testProcessBlockMerge3_cfg.py | 43 ++ ...testProcessBlockMergeOfMergedFiles2_cfg.py | 36 ++ .../testProcessBlockMergeOfMergedFiles_cfg.py | 77 ++++ .../test/testProcessBlockMerge_cfg.py | 87 ++++ ...estProcessBlockNOMergeOfMergedFiles_cfg.py | 72 +++ .../test/testProcessBlockNonStrict2_cfg.py | 83 ++++ .../test/testProcessBlockNonStrict3_cfg.py | 86 ++++ .../test/testProcessBlockNonStrict_cfg.py | 79 ++++ .../test/testProcessBlockRead2_cfg.py | 51 +++ .../testProcessBlockReadDropOnOutput2_cfg.py | 42 ++ .../testProcessBlockReadDropOnOutput_cfg.py | 42 ++ .../testProcessBlockReadThreeFileInput_cfg.py | 55 +++ .../test/testProcessBlockRead_cfg.py | 277 ++++++++++++ .../testProcessBlockSubProcessLooper_cfg.py | 219 ++++++++++ .../testProcessBlockSubProcessRead1_cfg.py | 76 ++++ .../testProcessBlockSubProcessRead2_cfg.py | 93 ++++ .../test/testProcessBlockSubProcess_cfg.py | 227 ++++++++++ .../test/testProcessBlockTEST_cfg.py | 81 ++++ .../testProcessBlockThreeFileInput_cfg.py | 70 +++ .../Integration/test/testProcessBlock_cfg.py | 14 - .../unit_test_outputs/testConsumesInfo_1.log | 79 +++- .../test/unit_test_outputs/testGetBy1.log | 28 ++ .../test/unit_test_outputs/testGetBy2.log | 28 ++ .../interface/ActivityRegistry.h | 16 + .../interface/ServiceRegistryfwd.h | 8 + .../ServiceRegistry/src/ActivityRegistry.cc | 6 + FWCore/Services/plugins/Tracer.cc | 17 + FWCore/Sources/interface/RawInputSource.h | 2 +- FWCore/Sources/src/RawInputSource.cc | 4 +- .../src/TFWLiteSelectorBasic.cc | 3 + .../TestProcessor/interface/TestProcessor.h | 2 + FWCore/TestProcessor/src/TestProcessor.cc | 7 +- FWCore/Utilities/interface/FWCoreUtiliesFwd.h | 9 + IOPool/Input/src/PoolSource.cc | 14 +- IOPool/Input/src/PoolSource.h | 5 +- IOPool/Input/src/RepeatingCachedRootSource.cc | 3 + IOPool/Input/src/RootDelayedReader.cc | 2 +- IOPool/Input/src/RootFile.cc | 373 +++++++++++++--- IOPool/Input/src/RootFile.h | 60 ++- IOPool/Input/src/RootInputFileSequence.cc | 15 + IOPool/Input/src/RootInputFileSequence.h | 3 + IOPool/Input/src/RootPrimaryFileSequence.cc | 155 +++++-- IOPool/Input/src/RootPrimaryFileSequence.h | 13 +- IOPool/Input/src/RootTree.cc | 111 +++-- IOPool/Input/src/RootTree.h | 82 ++-- .../PoolInputTest_skip_with_failure_cfg.py | 2 - IOPool/Output/interface/PoolOutputModule.h | 27 +- IOPool/Output/src/PoolOutputModule.cc | 64 ++- IOPool/Output/src/RootOutputFile.cc | 120 +++-- IOPool/Output/src/RootOutputFile.h | 23 +- IOPool/Output/src/RootOutputTree.cc | 7 +- IOPool/Output/src/RootOutputTree.h | 6 +- Mixing/Base/src/SecondaryEventProvider.cc | 4 +- 224 files changed, 9964 insertions(+), 1094 deletions(-) create mode 100644 DataFormats/Provenance/interface/EventToProcessBlockIndexes.h create mode 100644 DataFormats/Provenance/interface/StoredProcessBlockHelper.h create mode 100644 DataFormats/Provenance/src/StoredProcessBlockHelper.cc create mode 100644 DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp create mode 100644 FWCore/Common/interface/FWCoreCommonFwd.h create mode 100644 FWCore/Common/interface/OutputProcessBlockHelper.h create mode 100644 FWCore/Common/interface/ProcessBlockHelper.h create mode 100644 FWCore/Common/interface/ProcessBlockHelperBase.h create mode 100644 FWCore/Common/interface/SubProcessBlockHelper.h create mode 100644 FWCore/Common/src/OutputProcessBlockHelper.cc create mode 100644 FWCore/Common/src/ProcessBlockHelper.cc create mode 100644 FWCore/Common/src/ProcessBlockHelperBase.cc create mode 100644 FWCore/Common/src/SubProcessBlockHelper.cc create mode 100644 FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc create mode 100644 FWCore/Framework/interface/CacheHandle.h create mode 100644 FWCore/Framework/interface/InputProcessBlockCacheImpl.h create mode 100644 FWCore/Framework/interface/ProcessBlockForOutput.h create mode 100644 FWCore/Framework/interface/processBlockUtilities.h create mode 100644 FWCore/Framework/src/FileBlock.cc create mode 100644 FWCore/Framework/src/ProcessBlockForOutput.cc create mode 100644 FWCore/Framework/src/processBlockUtilities.cc create mode 100644 FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc create mode 100644 FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc create mode 100644 FWCore/Integration/test/TestOneOutput.cc create mode 100755 FWCore/Integration/test/run_TestProcessBlock.sh create mode 100644 FWCore/Integration/test/testLooperEventNavigation2.txt create mode 100644 FWCore/Integration/test/testLooperEventNavigation2_cfg.py create mode 100644 FWCore/Integration/test/testLooperEventNavigation3.txt create mode 100644 FWCore/Integration/test/testLooperEventNavigation3_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock1_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock2Dropped_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock3_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock4_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlock5_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockDropOnOutput2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockDropOnOutput_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockFailMerge_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockMerge2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockMerge3_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockMergeOfMergedFiles2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockMerge_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockNonStrict_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockRead2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockRead_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockSubProcess_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockTEST_cfg.py create mode 100644 FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py delete mode 100644 FWCore/Integration/test/testProcessBlock_cfg.py create mode 100644 FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h create mode 100644 FWCore/Utilities/interface/FWCoreUtiliesFwd.h diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index d58c08f32b1fe..73e1cf080b36f 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -325,7 +325,7 @@ class DQMRootSource : public edm::PuttableSourceBase, DQMTTreeIO { edm::InputSource::ItemType getNextItemType() override; - std::unique_ptr readFile_() override; + std::shared_ptr readFile_() override; std::shared_ptr readRunAuxiliary_() override; std::shared_ptr readLuminosityBlockAuxiliary_() override; void readRun_(edm::RunPrincipal& rpCache) override; @@ -465,7 +465,7 @@ DQMRootSource::~DQMRootSource() { edm::InputSource::ItemType DQMRootSource::getNextItemType() { return m_nextItemType; } // We will read the metadata of all files and fill m_fileMetadatas vector -std::unique_ptr DQMRootSource::readFile_() { +std::shared_ptr DQMRootSource::readFile_() { const int numFiles = m_catalog.fileNames(0).size(); m_openFiles.reserve(numFiles); @@ -601,7 +601,7 @@ std::unique_ptr DQMRootSource::readFile_() { m_nextItemType = edm::InputSource::IsRun; // We have to return something but not sure why - return std::make_unique(); + return std::make_shared(); } std::shared_ptr DQMRootSource::readRunAuxiliary_() { diff --git a/DataFormats/Provenance/interface/BranchType.h b/DataFormats/Provenance/interface/BranchType.h index d116fad2cc4dc..b93473a7efb3d 100644 --- a/DataFormats/Provenance/interface/BranchType.h +++ b/DataFormats/Provenance/interface/BranchType.h @@ -9,9 +9,12 @@ #include "FWCore/Utilities/interface/BranchType.h" namespace edm { + std::string const& BranchTypeToString(BranchType const& branchType); + constexpr unsigned int numberOfRunLumiEventProductTrees = 3; std::string const& BranchTypeToProductTreeName(BranchType const& branchType); + std::string BranchTypeToProductTreeName(BranchType const& branchType, std::string const& processName); std::string const& BranchTypeToMetaDataTreeName(BranchType const& branchType); @@ -49,6 +52,7 @@ namespace edm { // Other branches on Events Tree std::string const& eventSelectionsBranchName(); std::string const& branchListIndexesBranchName(); + std::string const& eventToProcessBlockIndexesBranchName(); //------------------------------------------------------------------ //------------------------------------------------------------------ @@ -70,6 +74,7 @@ namespace edm { std::string const& fileIndexBranchName(); // backward compatibility std::string const& indexIntoFileBranchName(); std::string const& mergeableRunProductMetadataBranchName(); + std::string const& processBlockHelperBranchName(); // Event History Tree // backward compatibility std::string const& eventHistoryTreeName(); // backward compatibility diff --git a/DataFormats/Provenance/interface/EventToProcessBlockIndexes.h b/DataFormats/Provenance/interface/EventToProcessBlockIndexes.h new file mode 100644 index 0000000000000..afbce387f4cda --- /dev/null +++ b/DataFormats/Provenance/interface/EventToProcessBlockIndexes.h @@ -0,0 +1,24 @@ +#ifndef DataFormats_Provenance_EventToProcessBlockIndexes_h +#define DataFormats_Provenance_EventToProcessBlockIndexes_h + +/** + +\author W. David Dagenhart, created 5 January, 2021 + +*/ + +namespace edm { + + class EventToProcessBlockIndexes { + public: + EventToProcessBlockIndexes() {} + + unsigned int index() const { return index_; } + void setIndex(unsigned int value) { index_ = value; } + + private: + unsigned int index_ = 0; + }; + +} // namespace edm +#endif diff --git a/DataFormats/Provenance/interface/ProvenanceFwd.h b/DataFormats/Provenance/interface/ProvenanceFwd.h index 942dcbf1924b1..2fb51b1adfc85 100644 --- a/DataFormats/Provenance/interface/ProvenanceFwd.h +++ b/DataFormats/Provenance/interface/ProvenanceFwd.h @@ -12,6 +12,7 @@ namespace edm { class ProductProvenance; class EventAuxiliary; class EventID; + class EventToProcessBlockIndexes; class LuminosityBlockAuxiliary; class LuminosityBlockID; class ModuleDescription; @@ -24,6 +25,7 @@ namespace edm { class RunAuxiliary; class RunID; class StableProvenance; + class StoredProcessBlockHelper; class Timestamp; class ProductProvenanceLookup; } // namespace edm diff --git a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h new file mode 100644 index 0000000000000..af906608ea4b1 --- /dev/null +++ b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h @@ -0,0 +1,50 @@ +#ifndef DataFormats_Provenance_StoredProcessBlockHelper_h +#define DataFormats_Provenance_StoredProcessBlockHelper_h + +/** \class edm::StoredProcessBlockHelper + +This contains the information from the ProcessBlockHelper +that is persistent. The processBlockCacheIndices_ data +member is a flattened version of the vector of vectors +in the ProcessBlockHelper. It is flattened mainly for +I/O performance reasons. This is intended to be directly +used only by the IOPool code responsible for reading +and writing the persistent files. Everything else should +interact with the ProcessBlockHelper. + +\author W. David Dagenhart, created 1 Oct, 2020 + +*/ + +#include +#include + +namespace edm { + + class StoredProcessBlockHelper { + public: + // This constructor exists for ROOT I/O + StoredProcessBlockHelper(); + + StoredProcessBlockHelper(std::vector const& processesWithProcessBlockProducts); + + std::vector const& processesWithProcessBlockProducts() const { + return processesWithProcessBlockProducts_; + } + + std::vector& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; } + + std::vector const& processBlockCacheIndices() const { return processBlockCacheIndices_; } + + std::vector& processBlockCacheIndices() { return processBlockCacheIndices_; } + + static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; } + static constexpr unsigned int invalidProcessIndex() { return 0xffffffff; } + + private: + std::vector processesWithProcessBlockProducts_; + + std::vector processBlockCacheIndices_; + }; +} // namespace edm +#endif diff --git a/DataFormats/Provenance/src/BranchType.cc b/DataFormats/Provenance/src/BranchType.cc index 68f074cd887f7..d85a381a80edc 100644 --- a/DataFormats/Provenance/src/BranchType.cc +++ b/DataFormats/Provenance/src/BranchType.cc @@ -78,11 +78,13 @@ namespace edm { std::string const fileIndex = "FileIndex"; std::string const indexIntoFile = "IndexIntoFile"; std::string const mergeableRunProductMetadata = "MergeableRunProductMetadata"; + std::string const processBlockHelper = "ProcessBlockHelper"; std::string const eventHistory = "EventHistory"; std::string const eventBranchMapper = "EventBranchMapper"; std::string const eventSelections = "EventSelections"; std::string const branchListIndexes = "BranchListIndexes"; + std::string const eventToProcessBlockIndexes = "EventToProcessBlockIndexes"; std::string const parameterSetsTree = "ParameterSets"; std::string const idToParameterSetBlobsBranch = "IdToParameterSetsBlobs"; @@ -95,6 +97,11 @@ namespace edm { return treeNames[branchType]; } + std::string BranchTypeToProductTreeName(BranchType const& branchType, std::string const& processName) { + assert(branchType == InProcess); + return branchTypeNames[InProcess] + "s" + processName; + } + std::string const& BranchTypeToMetaDataTreeName(BranchType const& branchType) { assert(branchType < eventLumiRunSize); return metaTreeNames[branchType]; @@ -193,6 +200,9 @@ namespace edm { // Branch on MetaData Tree std::string const& mergeableRunProductMetadataBranchName() { return mergeableRunProductMetadata; } + // Branch on MetaData Tree + std::string const& processBlockHelperBranchName() { return processBlockHelper; } + // Branch on Event History Tree std::string const& eventHistoryBranchName() { return eventHistory; } @@ -201,6 +211,8 @@ namespace edm { std::string const& branchListIndexesBranchName() { return branchListIndexes; } + std::string const& eventToProcessBlockIndexesBranchName() { return eventToProcessBlockIndexes; } + std::string const& parameterSetsTreeName() { return parameterSetsTree; } // Branch on ParameterSets Tree std::string const& idToParameterSetBlobsBranchName() { return idToParameterSetBlobsBranch; } diff --git a/DataFormats/Provenance/src/ProductRegistry.cc b/DataFormats/Provenance/src/ProductRegistry.cc index 5dbcd799d7233..04f478c06f4f1 100644 --- a/DataFormats/Provenance/src/ProductRegistry.cc +++ b/DataFormats/Provenance/src/ProductRegistry.cc @@ -273,7 +273,8 @@ namespace edm { } ++i; } else if (i == e || (j != s && j->first < i->first)) { - if (j->second.present() && branchesMustMatch == BranchDescription::Strict) { + if (j->second.present() && + (branchesMustMatch == BranchDescription::Strict || j->second.branchType() == InProcess)) { differences << "Branch '" << j->second.branchName() << "' is in previous files\n"; differences << " but not in file '" << fileName << "'.\n"; } diff --git a/DataFormats/Provenance/src/StoredProcessBlockHelper.cc b/DataFormats/Provenance/src/StoredProcessBlockHelper.cc new file mode 100644 index 0000000000000..2814eb2e2c5de --- /dev/null +++ b/DataFormats/Provenance/src/StoredProcessBlockHelper.cc @@ -0,0 +1,10 @@ +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" + +namespace edm { + + StoredProcessBlockHelper::StoredProcessBlockHelper() {} + + StoredProcessBlockHelper::StoredProcessBlockHelper(std::vector const& processesWithProcessBlockProducts) + : processesWithProcessBlockProducts_(processesWithProcessBlockProducts) {} + +} // namespace edm diff --git a/DataFormats/Provenance/src/classes.h b/DataFormats/Provenance/src/classes.h index 540983d9c574f..f1c21e84d8e45 100644 --- a/DataFormats/Provenance/src/classes.h +++ b/DataFormats/Provenance/src/classes.h @@ -5,6 +5,7 @@ #include "DataFormats/Provenance/interface/ElementID.h" #include "DataFormats/Provenance/interface/EventAuxiliary.h" #include "DataFormats/Provenance/interface/EventID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/FileFormatVersion.h" #include "DataFormats/Provenance/interface/FileID.h" #include "DataFormats/Provenance/interface/FileIndex.h" @@ -27,6 +28,7 @@ #include "DataFormats/Provenance/interface/ProductID.h" #include "DataFormats/Provenance/interface/ProductProvenance.h" #include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" #include "DataFormats/Provenance/interface/StoredProductProvenance.h" #include "DataFormats/Provenance/interface/ProductRegistry.h" #include "DataFormats/Provenance/interface/RunAuxiliary.h" diff --git a/DataFormats/Provenance/src/classes_def.xml b/DataFormats/Provenance/src/classes_def.xml index 0806756b3df4d..0cd4c91e3409d 100644 --- a/DataFormats/Provenance/src/classes_def.xml +++ b/DataFormats/Provenance/src/classes_def.xml @@ -147,6 +147,9 @@ + + + @@ -217,6 +220,10 @@ + + + + initializeTransients(); diff --git a/DataFormats/Provenance/test/BuildFile.xml b/DataFormats/Provenance/test/BuildFile.xml index 660587cd4b351..b2e942986b36a 100644 --- a/DataFormats/Provenance/test/BuildFile.xml +++ b/DataFormats/Provenance/test/BuildFile.xml @@ -10,6 +10,10 @@ + + + + diff --git a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp new file mode 100644 index 0000000000000..504a0372f7b4e --- /dev/null +++ b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp @@ -0,0 +1,46 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" + +#include +#include +#include + +TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { + SECTION("Default construction") { + edm::StoredProcessBlockHelper storedProcessBlockHelper; + REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts().empty()); + REQUIRE(storedProcessBlockHelper.processBlockCacheIndices().empty()); + + edm::StoredProcessBlockHelper const* storedProcessBlockHelperConstPtr = &storedProcessBlockHelper; + REQUIRE(storedProcessBlockHelperConstPtr->processesWithProcessBlockProducts().empty()); + REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); + + REQUIRE(edm::StoredProcessBlockHelper::invalidCacheIndex() == 0xffffffff); + REQUIRE(edm::StoredProcessBlockHelper::invalidProcessIndex() == 0xffffffff); + + edm::EventToProcessBlockIndexes eventToProcessBlockIndexes; + REQUIRE(eventToProcessBlockIndexes.index() == 0); + eventToProcessBlockIndexes.setIndex(2); + REQUIRE(eventToProcessBlockIndexes.index() == 2); + } + + SECTION("Constructor") { + std::vector testStrings{std::string("test1"), std::string("test2"), std::string("test3")}; + + edm::StoredProcessBlockHelper storedProcessBlockHelper(testStrings); + REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts() == testStrings); + REQUIRE(storedProcessBlockHelper.processBlockCacheIndices().empty()); + + edm::StoredProcessBlockHelper const* storedProcessBlockHelperConstPtr = &storedProcessBlockHelper; + REQUIRE(storedProcessBlockHelperConstPtr->processesWithProcessBlockProducts() == testStrings); + REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); + + std::vector testIndices{1, 10, 100}; + storedProcessBlockHelper.processBlockCacheIndices() = testIndices; + REQUIRE(storedProcessBlockHelper.processBlockCacheIndices() == testIndices); + REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices() == testIndices); + } +} diff --git a/FWCore/Common/interface/FWCoreCommonFwd.h b/FWCore/Common/interface/FWCoreCommonFwd.h new file mode 100644 index 0000000000000..73adc408c5e8f --- /dev/null +++ b/FWCore/Common/interface/FWCoreCommonFwd.h @@ -0,0 +1,12 @@ +#ifndef FWCore_Common_FWCoreCommonFwd_h +#define FWCore_Common_FWCoreCommonFwd_h + +namespace edm { + + class ProcessBlockHelper; + class ProcessBlockHelperBase; + class SubProcessBlockHelper; + +} // namespace edm + +#endif diff --git a/FWCore/Common/interface/OutputProcessBlockHelper.h b/FWCore/Common/interface/OutputProcessBlockHelper.h new file mode 100644 index 0000000000000..7ee5bd413cd50 --- /dev/null +++ b/FWCore/Common/interface/OutputProcessBlockHelper.h @@ -0,0 +1,76 @@ +#ifndef FWCore_Common_OutputProcessBlockHelper_h +#define FWCore_Common_OutputProcessBlockHelper_h + +/** \class edm::OutputProcessBlockHelper + +\author W. David Dagenhart, created 28 December, 2020 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" + +#include +#include +#include + +namespace edm { + + class ProcessBlockHelperBase; + + class OutputProcessBlockHelper { + public: + std::vector const& processesWithProcessBlockProducts() const { + return processesWithProcessBlockProducts_; + } + + void updateAfterProductSelection(std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelperBase const&); + + void fillCacheIndices(StoredProcessBlockHelper&) const; + + bool productsFromInputKept() const { return productsFromInputKept_; } + + static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; } + + ProcessBlockHelperBase const* processBlockHelper() const { return processBlockHelper_; } + + // These are intended to be used only for testing purposes + std::vector const& translateFromStoredIndex() const { return translateFromStoredIndex_; } + unsigned int nAddedProcesses() const { return nAddedProcesses_; } + + private: + void setStoredProcessOffset(unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& storedProcessOffset) const; + + void setProcessOffset(unsigned int iFile, + unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& processOffset) const; + + void setStoredFileInProcessOffset(unsigned int iFile, + unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& storedFileInProcessOffset) const; + + // Includes processes with at least one ProcessBlock branch present + // in the output file + std::vector processesWithProcessBlockProducts_; + + // This will have the value of 0 or 1, except for the SubProcess case. + // This is incremented to 1 if the current process produces new + // ProcessBlock products and they are kept by the OutputModule. + unsigned int nAddedProcesses_ = 0; + + // Translate from the vector of process names in this class to + // the one in the ProcessBlockHelper + std::vector translateFromStoredIndex_; + + // Points to the main ProcessBlockHelper owned by the EventProcessor + // or SubProcess + ProcessBlockHelperBase const* processBlockHelper_ = nullptr; + + bool productsFromInputKept_ = false; + }; +} // namespace edm +#endif diff --git a/FWCore/Common/interface/ProcessBlockHelper.h b/FWCore/Common/interface/ProcessBlockHelper.h new file mode 100644 index 0000000000000..176dc6110faca --- /dev/null +++ b/FWCore/Common/interface/ProcessBlockHelper.h @@ -0,0 +1,120 @@ +#ifndef FWCore_Common_ProcessBlockHelper_h +#define FWCore_Common_ProcessBlockHelper_h + +/** \class edm::ProcessBlockHelper + +\author W. David Dagenhart, created 15 September, 2020 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" + +#include +#include +#include + +namespace edm { + + class ProcessBlockHelper : public ProcessBlockHelperBase { + public: + ProcessBlockHelperBase const* topProcessBlockHelper() const override; + std::vector const& topProcessesWithProcessBlockProducts() const override; + unsigned int nProcessesInFirstFile() const override; + std::vector> const& processBlockCacheIndices() const override; + std::vector> const& nEntries() const override; + std::vector const& cacheIndexVectorsPerFile() const override; + std::vector const& cacheEntriesPerFile() const override; + unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const override; + unsigned int outerOffset() const override; + + bool initializedFromInput() const { return initializedFromInput_; } + + bool firstFileDropProcessesAndReorderStored(std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::set const& processesToKeep, + std::vector const& nEntries, + std::vector& finalIndexToStoredIndex) const; + + bool dropProcessesAndReorderStored(std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::set const& processesToKeep, + std::vector const& nEntries, + std::vector& finalIndexToStoredIndex, + std::vector const& firstFileFinalProcesses) const; + + void initializeFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper); + + void fillFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper, + std::vector&& nEntries); + + void clearAfterOutputFilesClose(); + + private: + void dropProcessesAndReorderStoredImpl(std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::vector& finalProcesses, + std::vector const& nEntries, + std::vector const& finalIndexToStoredIndex) const; + + void fillFromPrimaryInputWhenNotEmpty(std::vector const& storedProcesses, + std::vector const& storedCacheIndices, + std::vector&& nEntries); + + void fillEntriesFromPrimaryInput(std::vector&& nEntries); + + // A general comment about this class and its data members. + // It was initially written to handle cases where all ProcessBlock + // products from some process were dropped in a file after + // the first input file but were present in the first input file. + // At the moment this comment is being written, the file merging + // rules do not allow this to happen and this situation never + // occurs. However, this class intentionally maintains support + // for this case, because we may find we need to change the file + // merging requirements in the future. So there is support for + // some indices to be invalid or other values to be zero even + // though at the moment this should never occur. + + // Events hold an index into the outer vector + // (an offset needs to added in the case of multiple input + // files). The elements of the inner vector correspond to the + // processes in processesWithProcessBlockProducts_ (exactly + // 1 to 1 in the same order except it only includes those processes + // from the input, if the current Process and/or SubProcesses are + // added, then they are added to the container of cache indices when + // the output module makes its modified copy). The values inside + // the inner vector are the cache indices into the cache vectors + // contained by user modules. This cache order is the same as the + // processing order of ProcessBlocks in the current process. + // It might contain invalid cache index values. + std::vector> processBlockCacheIndices_; + + // Number of entries per ProcessBlock TTree. + // The outer vector has an element for each input file. + // The inner vector elements correspond 1-to-1 with + // processesWithProcessBlockProducts_ and in the same + // order. This might contain zeroes. + std::vector> nEntries_; + + // The index into the next two vectors is the input file index + // which is based on the order input files are read + // These can contain zeroes. + std::vector cacheIndexVectorsPerFile_; + std::vector cacheEntriesPerFile_; + + unsigned int nProcessesInFirstFile_ = 0; + + bool initializedFromInput_ = false; + + // Index of the first outer vector element in the cache indices + // container that is associated with the current input file. If + // it points to the end, then there were no cache indices in the + // current input file. + unsigned int outerOffset_ = 0; + + // Offset for the cacheIndex, counts all entries in + // ProcessBlock TTree's in all input files seen so far. + unsigned int cacheIndexOffset_ = 0; + }; +} // namespace edm +#endif diff --git a/FWCore/Common/interface/ProcessBlockHelperBase.h b/FWCore/Common/interface/ProcessBlockHelperBase.h new file mode 100644 index 0000000000000..ba98b9f40a5e3 --- /dev/null +++ b/FWCore/Common/interface/ProcessBlockHelperBase.h @@ -0,0 +1,67 @@ +#ifndef FWCore_Common_ProcessBlockHelperBase_h +#define FWCore_Common_ProcessBlockHelperBase_h + +/** \class edm::ProcessBlockHelperBase + +\author W. David Dagenhart, created 30 December, 2020 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Utilities/interface/FWCoreUtiliesFwd.h" + +#include +#include + +namespace edm { + + class ProcessBlockHelperBase { + public: + virtual ~ProcessBlockHelperBase() = default; + + std::vector& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; } + std::vector const& processesWithProcessBlockProducts() const { + return processesWithProcessBlockProducts_; + } + + std::vector& addedProcesses() { return addedProcesses_; } + std::vector const& addedProcesses() const { return addedProcesses_; } + + void updateForNewProcess(ProductRegistry const&, std::string const& processName); + + // In the function names below, top implies associated the helper associated + // with the EventProcessor, not a helper associated with a SubProcess. + + virtual ProcessBlockHelperBase const* topProcessBlockHelper() const = 0; + virtual std::vector const& topProcessesWithProcessBlockProducts() const = 0; + virtual unsigned int nProcessesInFirstFile() const = 0; + virtual std::vector> const& processBlockCacheIndices() const = 0; + virtual std::vector> const& nEntries() const = 0; + virtual std::vector const& cacheIndexVectorsPerFile() const = 0; + virtual std::vector const& cacheEntriesPerFile() const = 0; + virtual unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const = 0; + virtual unsigned int outerOffset() const = 0; + + std::string selectProcess(ProductRegistry const&, ProductLabels const&, TypeID const&) const; + + static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; } + static constexpr unsigned int invalidProcessIndex() { return 0xffffffff; } + + private: + // Includes processes with ProcessBlock branches present + // in the first input file and not dropped on input. At + // each processing step the new process will be added at + // the end if there are non-transient ProcessBlock products + // being produced. Output modules will write a copy of this + // to persistent storage after removing any process without + // at least one kept ProcessBlock branch. + std::vector processesWithProcessBlockProducts_; + + // This will have 0 or 1 element depending whether there are any + // non-transient ProcessBlock products produced in the current + // process (except for SubProcesses where this might have more + // than 1 element) + std::vector addedProcesses_; + }; +} // namespace edm +#endif diff --git a/FWCore/Common/interface/SubProcessBlockHelper.h b/FWCore/Common/interface/SubProcessBlockHelper.h new file mode 100644 index 0000000000000..ed17a83aa4069 --- /dev/null +++ b/FWCore/Common/interface/SubProcessBlockHelper.h @@ -0,0 +1,35 @@ +#ifndef FWCore_Common_SubProcessBlockHelper_h +#define FWCore_Common_SubProcessBlockHelper_h + +/** \class edm::SubProcessBlockHelper + +\author W. David Dagenhart, created 4 January, 2021 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" + +#include + +namespace edm { + + class SubProcessBlockHelper : public ProcessBlockHelperBase { + public: + ProcessBlockHelperBase const* topProcessBlockHelper() const override; + std::vector const& topProcessesWithProcessBlockProducts() const override; + unsigned int nProcessesInFirstFile() const override; + std::vector> const& processBlockCacheIndices() const override; + std::vector> const& nEntries() const override; + std::vector const& cacheIndexVectorsPerFile() const override; + std::vector const& cacheEntriesPerFile() const override; + unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const override; + unsigned int outerOffset() const override; + + void updateFromParentProcess(ProcessBlockHelperBase const& parentProcessBlockHelper, ProductRegistry const&); + + private: + ProcessBlockHelperBase const* topProcessBlockHelper_ = nullptr; + }; +} // namespace edm +#endif diff --git a/FWCore/Common/src/OutputProcessBlockHelper.cc b/FWCore/Common/src/OutputProcessBlockHelper.cc new file mode 100644 index 0000000000000..70de57d56e5fd --- /dev/null +++ b/FWCore/Common/src/OutputProcessBlockHelper.cc @@ -0,0 +1,231 @@ +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" + +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" + +#include +#include + +namespace edm { + + void OutputProcessBlockHelper::updateAfterProductSelection( + std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelperBase const& processBlockHelper) { + processBlockHelper_ = &processBlockHelper; + + // Copy the list of processes with ProcessBlock products from the EventProcessor or SubProcess, + // except remove any processes where the output module calling this has dropped all ProcessBlock + // products. We want to maintain the same order and only remove elements. Fill a vector that can + // translate from one process list to the other. + assert(processesWithProcessBlockProducts_.empty()); + unsigned int iProcess = 0; + for (auto const& process : processBlockHelper.processesWithProcessBlockProducts()) { + if (processesWithKeptProcessBlockProducts.find(process) != processesWithKeptProcessBlockProducts.end()) { + processesWithProcessBlockProducts_.emplace_back(process); + translateFromStoredIndex_.emplace_back(iProcess); + } + ++iProcess; + } + + for (auto const& addedProcess : processBlockHelper.addedProcesses()) { + // count new processes producing ProcessBlock products that are + // kept by the OutputModule. There can be at most 1 except + // in the case of SubProcesses. + if (std::find(processesWithProcessBlockProducts_.begin(), + processesWithProcessBlockProducts_.end(), + addedProcess) != processesWithProcessBlockProducts_.end()) { + ++nAddedProcesses_; + } + } + + // Determine if any ProcessBlock product from the input file is kept by the output module. + // Do this by looking for a process name on both the list of processes with ProcessBlock + // products kept by the output module and process names from the input with ProcessBlock + // products. + productsFromInputKept_ = + std::find_first_of(processesWithProcessBlockProducts_.begin(), + processesWithProcessBlockProducts_.end(), + processBlockHelper.topProcessesWithProcessBlockProducts().begin(), + processBlockHelper.topProcessesWithProcessBlockProducts().begin() + + processBlockHelper.nProcessesInFirstFile()) != processesWithProcessBlockProducts_.end(); + } + + void OutputProcessBlockHelper::fillCacheIndices(StoredProcessBlockHelper& storedProcessBlockHelper) const { + // The stored cache indices are the ones we want to fill. + // This will get written to the output file. + // Note for output the vector of vectors is flattened into a single vector + std::vector& storedCacheIndices = storedProcessBlockHelper.processBlockCacheIndices(); + + // Number of processes in StoredProcessBlockHelper. + unsigned int nStoredProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts().size(); + + if (!productsFromInputKept_) { + // This is really simple if we are not keeping any ProcessBlock products + // from the input file. Deal with that special case first. + // Except for the special case of a SubProcess, nStoredProcesses will be 1. + assert(nAddedProcesses_ == nStoredProcesses); + storedCacheIndices.reserve(nStoredProcesses); + for (unsigned int i = 0; i < nStoredProcesses; ++i) { + storedCacheIndices.push_back(i); + } + return; + } + + // Cache indices of the main ProcessBlockHelper we use as input. This + // ProcessBlockHelper is owned by the EventProcessor. + std::vector> const& cacheIndices = processBlockHelper_->processBlockCacheIndices(); + + // We need to convert the cache indices in the ProcessBlockHelper to have different values when + // put in the StoredProcessBlockHelper. The values are not the same because the ProcessBlocks are + // not read in the same order in this process as compared to the next process which will read + // the output file that is being written (the read order is the same as the order the cache + // objects are placed in the cache vectors). In this process, they are ordered first by input file, + // second by process and last by TTree entry. In the next process, this output file will be read + // as a single input file. The ProcessBlocks are read in process order (this will be a subset + // of the process list in ProcessBlockHelper, maybe smaller or maybe the same), and finally in + // order of entry in the TTree. This conversion is done by subtracting and adding some + // offsets and a lot of the following code involves calculating these offsets to do the conversion. + + // We will need the info in these to calculate the offsets + std::vector const& cacheIndexVectorsPerFile = processBlockHelper_->cacheIndexVectorsPerFile(); + std::vector const& cacheEntriesPerFile = processBlockHelper_->cacheEntriesPerFile(); + std::vector> const& nEntries = processBlockHelper_->nEntries(); + + assert(!cacheIndices.empty()); + // Count processes in the input file with saved ProcessBlock products in the output + unsigned int nInputProcesses = 0; + for (unsigned int iStoredProcess = 0; iStoredProcess < nStoredProcesses; ++iStoredProcess) { + // The existing cache indices in processBlockHelper include only indices + // corresponding to the processes in the input files. If there are more, then + // they correspond to current process (and there only will be more if some + // of the newly produced ProcessBlock products are going to be kept). + // There will be at most 1 added process except in the case of SubProcesses. + if (translateFromStoredIndex_[iStoredProcess] < cacheIndices[0].size()) { + ++nInputProcesses; + } + } + + // The following are the 4 offsets. The first two are defined relative to the + // cache sequence in this process. The second two offsets are defined relative + // to the cache sequence when the output file we are writing is read. + + // 1. Total number of cache entries in all input files prior to the current input file + unsigned int fileOffset = 0; + + // 2. For each process, the total number of cache entries in processes in the current + // input file and before the process + std::vector processOffset(nInputProcesses, 0); + + // 3. For each input process with ProcessBlock products stored by this + // output module, the total number of cache entries in earlier input processes + // that have ProcessBlock products stored by this output module. + // Summed over all input files and including only processes in StoredProcessBlockHelper. + // Plus an extra element at the end that includes all entries in all such processes. + assert(!nEntries.empty()); + std::vector storedProcessOffset(nInputProcesses + 1, 0); + + // 4. For each process with ProcessBlock products stored by this output module, + // the total number of cache entries in that process in all input files before + // the current input file. + std::vector storedFileInProcessOffset(nInputProcesses, 0); + + setStoredProcessOffset(nInputProcesses, nEntries, storedProcessOffset); + + storedCacheIndices.reserve(cacheIndices.size() * nStoredProcesses); + + unsigned int iFile = 0; + unsigned int innerVectorsCurrentFile = 0; + + // In ProcessBlockHelper, there is a vector which contains vectors + // of cache indices. Iterate over the inner vectors. + for (auto const& innerVectorOfCacheIndices : cacheIndices) { + // The inner vectors are associated with input files. Several contiguous + // inner vectors can be associated with the same input file. Check to + // see if we have crossed an input file boundary and set the file + // index, iFile, at the next file associated with at least + // one inner vector if necessary. + while (innerVectorsCurrentFile == cacheIndexVectorsPerFile[iFile]) { + // Sum cache entries for all files before the current file in + // ProcessBlockHelper + fileOffset += cacheEntriesPerFile[iFile]; + ++iFile; + innerVectorsCurrentFile = 0; + } + if (innerVectorsCurrentFile == 0) { + // Call these when the input file changes + setProcessOffset(iFile, nInputProcesses, nEntries, processOffset); + setStoredFileInProcessOffset(iFile, nInputProcesses, nEntries, storedFileInProcessOffset); + } + ++innerVectorsCurrentFile; + + assert(nInputProcesses + nAddedProcesses_ == nStoredProcesses); + + // Really fill the cache indices that will be stored in the output file in this loop + for (unsigned int iStoredProcess = 0; iStoredProcess < nStoredProcesses; ++iStoredProcess) { + unsigned int storedCacheIndex = invalidCacheIndex(); + if (iStoredProcess < nInputProcesses) { + unsigned int cacheIndex = innerVectorOfCacheIndices[translateFromStoredIndex_[iStoredProcess]]; + if (cacheIndex != invalidCacheIndex()) { + // The offsets count in the cache sequence to the first entry in + // the current input file and process + unsigned int inputOffset = fileOffset + processOffset[iStoredProcess]; + unsigned int storedOffset = storedProcessOffset[iStoredProcess] + storedFileInProcessOffset[iStoredProcess]; + storedCacheIndex = cacheIndex - inputOffset + storedOffset; + } + } else { + // This corresponds to the current process if it has newly produced + // ProcessBlock products (plus possibly SubProcesses). + storedCacheIndex = storedProcessOffset[nInputProcesses] + iStoredProcess - nInputProcesses; + } + storedCacheIndices.push_back(storedCacheIndex); + } + } + } + + void OutputProcessBlockHelper::setStoredProcessOffset(unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& storedProcessOffset) const { + for (unsigned int iStored = 0; iStored < nInputProcesses + 1; ++iStored) { + storedProcessOffset[iStored] = 0; + // loop over earlier processes + for (unsigned int jStored = 0; jStored < iStored; ++jStored) { + unsigned int indexInEventProcessor = translateFromStoredIndex_[jStored]; + // loop over input files + for (auto const& entries : nEntries) { + assert(indexInEventProcessor < entries.size()); + storedProcessOffset[iStored] += entries[indexInEventProcessor]; + } + } + } + } + + void OutputProcessBlockHelper::setProcessOffset(unsigned int iFile, + unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& processOffset) const { + for (unsigned int iStored = 0; iStored < nInputProcesses; ++iStored) { + processOffset[iStored] = 0; + unsigned int iProcess = translateFromStoredIndex_[iStored]; + for (unsigned int jProcess = 0; jProcess < iProcess; ++jProcess) { + processOffset[iStored] += nEntries[iFile][jProcess]; + } + } + } + + void OutputProcessBlockHelper::setStoredFileInProcessOffset( + unsigned int iFile, + unsigned int nInputProcesses, + std::vector> const& nEntries, + std::vector& storedFileInProcessOffset) const { + for (unsigned int iStored = 0; iStored < nInputProcesses; ++iStored) { + storedFileInProcessOffset[iStored] = 0; + unsigned int indexInEventProcessor = translateFromStoredIndex_[iStored]; + // loop over input files before current input file + for (unsigned int jFile = 0; jFile < iFile; ++jFile) { + assert(indexInEventProcessor < nEntries[jFile].size()); + storedFileInProcessOffset[iStored] += nEntries[jFile][indexInEventProcessor]; + } + } + } + +} // namespace edm diff --git a/FWCore/Common/src/ProcessBlockHelper.cc b/FWCore/Common/src/ProcessBlockHelper.cc new file mode 100644 index 0000000000000..56e14a71e9c30 --- /dev/null +++ b/FWCore/Common/src/ProcessBlockHelper.cc @@ -0,0 +1,279 @@ +#include "FWCore/Common/interface/ProcessBlockHelper.h" + +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" + +#include +#include +#include + +namespace edm { + + ProcessBlockHelperBase const* ProcessBlockHelper::topProcessBlockHelper() const { return this; } + + std::vector const& ProcessBlockHelper::topProcessesWithProcessBlockProducts() const { + return processesWithProcessBlockProducts(); + } + + unsigned int ProcessBlockHelper::nProcessesInFirstFile() const { return nProcessesInFirstFile_; } + + std::vector> const& ProcessBlockHelper::processBlockCacheIndices() const { + return processBlockCacheIndices_; + } + + std::vector> const& ProcessBlockHelper::nEntries() const { return nEntries_; } + + std::vector const& ProcessBlockHelper::cacheIndexVectorsPerFile() const { + return cacheIndexVectorsPerFile_; + } + + std::vector const& ProcessBlockHelper::cacheEntriesPerFile() const { return cacheEntriesPerFile_; } + + unsigned int ProcessBlockHelper::processBlockIndex( + std::string const& processName, EventToProcessBlockIndexes const& eventToProcessBlockIndexes) const { + for (unsigned int iProcess = 0; iProcess < nProcessesInFirstFile_; ++iProcess) { + if (processName == processesWithProcessBlockProducts()[iProcess]) { + return processBlockCacheIndices_[eventToProcessBlockIndexes.index()][iProcess]; + } + } + return invalidCacheIndex(); + } + + unsigned int ProcessBlockHelper::outerOffset() const { return outerOffset_; } + + // Modifies the process names and cache indices in the StoredProcessBlockHelper. + // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks + // to be read in the same order for all input files. + bool ProcessBlockHelper::firstFileDropProcessesAndReorderStored( + std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::set const& processesToKeep, + std::vector const& nEntries, + std::vector& finalIndexToStoredIndex) const { + bool isModified = false; + unsigned int finalSize = 0; + for (auto const& process : storedProcesses) { + if (processesToKeep.find(process) == processesToKeep.end()) { + isModified = true; + } else { + ++finalSize; + } + } + if (!isModified) { + return isModified; + } + + std::vector finalProcesses; + finalProcesses.reserve(finalSize); + finalIndexToStoredIndex.reserve(finalSize); + for (unsigned int iStored = 0; iStored < storedProcesses.size(); ++iStored) { + if (processesToKeep.find(storedProcesses[iStored]) != processesToKeep.end()) { + finalProcesses.emplace_back(storedProcesses[iStored]); + finalIndexToStoredIndex.emplace_back(iStored); + } + } + dropProcessesAndReorderStoredImpl( + storedProcesses, storedCacheIndices, finalProcesses, nEntries, finalIndexToStoredIndex); + return isModified; + } + + // Modifies the process names and cache indices in the StoredProcessBlockHelper. + // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks + // to be read in the same order for all input files. + bool ProcessBlockHelper::dropProcessesAndReorderStored(std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::set const& processesToKeep, + std::vector const& nEntries, + std::vector& finalIndexToStoredIndex, + std::vector const& firstFileFinalProcesses) const { + std::vector finalProcesses; + // Always will allocate enough space (sometimes too much) + finalProcesses.reserve(storedProcesses.size()); + finalIndexToStoredIndex.reserve(storedProcesses.size()); + + // The outer loop here establishes the order + // Only allows processes that got into finalProcesses for the first file + for (unsigned int iFirst = 0; iFirst < firstFileFinalProcesses.size(); ++iFirst) { + // Only includes processes also in storedProcesses + for (unsigned int iStored = 0; iStored < storedProcesses.size(); ++iStored) { + std::string const& storedProcess = storedProcesses[iStored]; + if (firstFileFinalProcesses[iFirst] == storedProcess) { + // Also requires processes have kept ProcessBlock products + if (processesToKeep.find(storedProcess) != processesToKeep.end()) { + finalProcesses.emplace_back(storedProcess); + finalIndexToStoredIndex.emplace_back(iStored); + break; + } + } + } + } + + bool isModified = true; + if (storedProcesses == finalProcesses) { + isModified = false; + return isModified; + } + + dropProcessesAndReorderStoredImpl( + storedProcesses, storedCacheIndices, finalProcesses, nEntries, finalIndexToStoredIndex); + return isModified; + } + + void ProcessBlockHelper::initializeFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper) { + if (!initializedFromInput_) { + initializedFromInput_ = true; + + assert(processesWithProcessBlockProducts().empty()); + processesWithProcessBlockProducts() = storedProcessBlockHelper.processesWithProcessBlockProducts(); + nProcessesInFirstFile_ = processesWithProcessBlockProducts().size(); + } + } + + void ProcessBlockHelper::fillFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper, + std::vector&& nEntries) { + // I've written this so it will continue to work even if we someday relax + // the strict merging requirement in the ProductRegistry (there + // is a little extra complexity that may be unnecessary...). + + std::vector const& storedProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts(); + std::vector const& storedCacheIndices = storedProcessBlockHelper.processBlockCacheIndices(); + + outerOffset_ = processBlockCacheIndices_.size(); + + if (nProcessesInFirstFile_ == 0) { + // There were no ProcessBlock products in the first file. Nothing to do. + return; + } else if (!storedProcesses.empty()) { + // Subsequent input file with ProcessBlock products + fillFromPrimaryInputWhenNotEmpty(storedProcesses, storedCacheIndices, std::move(nEntries)); + } else if (storedProcesses.empty()) { + // Subsequent input file without ProcessBlock products + processBlockCacheIndices_.emplace_back(nProcessesInFirstFile_, invalidCacheIndex()); + cacheIndexVectorsPerFile_.push_back(1); + std::vector newNEntries(nProcessesInFirstFile_, 0); + fillEntriesFromPrimaryInput(std::move(newNEntries)); + } + } + + void ProcessBlockHelper::clearAfterOutputFilesClose() { + processBlockCacheIndices_.clear(); + nEntries_.clear(); + cacheIndexVectorsPerFile_.clear(); + cacheEntriesPerFile_.clear(); + outerOffset_ = 0; + cacheIndexOffset_ = 0; + } + + // Modifies the process names and cache indices in the StoredProcessBlockHelper. + // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks + // to be read in the same order for all input files. + void ProcessBlockHelper::dropProcessesAndReorderStoredImpl( + std::vector& storedProcesses, + std::vector& storedCacheIndices, + std::vector& finalProcesses, + std::vector const& nEntries, + std::vector const& finalIndexToStoredIndex) const { + assert(!storedProcesses.empty()); + std::vector newCacheIndices; + if (!finalProcesses.empty()) { + // ProcessBlocks are read in the order of storedProcesses and within + // each process in entry order in the TTree. This establishes the cache + // order that the values in storedCacheIndices refer to. The "offset" refers + // to cache index value of the first ProcessBlock in a TTree. + std::vector oldOffsets; + oldOffsets.reserve(storedProcesses.size()); + unsigned int offset = 0; + for (auto const& entries : nEntries) { + oldOffsets.emplace_back(offset); + offset += entries; + } + + unsigned int nFinalProcesses = finalProcesses.size(); + std::vector newOffsets; + newOffsets.reserve(nFinalProcesses); + offset = 0; + for (unsigned int iNew = 0; iNew < nFinalProcesses; ++iNew) { + newOffsets.emplace_back(offset); + offset += nEntries[finalIndexToStoredIndex[iNew]]; + } + + unsigned int nOuterLoop = storedCacheIndices.size() / storedProcesses.size(); + assert(nOuterLoop * storedProcesses.size() == storedCacheIndices.size()); + newCacheIndices.reserve(nOuterLoop * nFinalProcesses); + unsigned int storedOuterOffset = 0; + + for (unsigned int k = 0; k < nOuterLoop; ++k) { + for (unsigned int j = 0; j < nFinalProcesses; ++j) { + unsigned int storedIndex = finalIndexToStoredIndex[j]; + unsigned int oldCacheIndex = storedCacheIndices[storedOuterOffset + storedIndex]; + unsigned int newCacheIndex = StoredProcessBlockHelper::invalidCacheIndex(); + if (oldCacheIndex != StoredProcessBlockHelper::invalidCacheIndex()) { + newCacheIndex = oldCacheIndex - oldOffsets[storedIndex] + newOffsets[j]; + } + newCacheIndices.emplace_back(newCacheIndex); + } + storedOuterOffset += storedProcesses.size(); + } + } + storedCacheIndices.swap(newCacheIndices); + storedProcesses.swap(finalProcesses); + } + + void ProcessBlockHelper::fillFromPrimaryInputWhenNotEmpty(std::vector const& storedProcesses, + std::vector const& storedCacheIndices, + std::vector&& nEntries) { + assert(nProcessesInFirstFile_ <= processesWithProcessBlockProducts().size()); + + // Calculate a translation from an index into the process names from the first file + // to an index into the process names from the current file. Note the value will + // be left invalid if the process name is not found in the current file. + std::vector firstFileToStored(nProcessesInFirstFile_, invalidProcessIndex()); + std::vector newNEntries(nProcessesInFirstFile_, 0); + for (unsigned int j = 0; j < nProcessesInFirstFile_; ++j) { + for (unsigned int k = 0; k < storedProcesses.size(); ++k) { + if (processesWithProcessBlockProducts()[j] == storedProcesses[k]) { + firstFileToStored[j] = k; + newNEntries[j] = nEntries[k]; + break; + } + } + } + + // Append the cache indices from the current input file to the + // cache indices container from the previous files. + unsigned int nCacheIndexVectors = storedCacheIndices.size() / storedProcesses.size(); + assert(storedProcesses.size() * nCacheIndexVectors == storedCacheIndices.size()); + processBlockCacheIndices_.resize(processBlockCacheIndices_.size() + nCacheIndexVectors); + unsigned int storedIndex = 0; + for (unsigned int k = 0; k < nCacheIndexVectors; ++k) { + processBlockCacheIndices_[outerOffset_ + k].reserve(nProcessesInFirstFile_); + for (unsigned int j = 0; j < nProcessesInFirstFile_; ++j) { + unsigned int iStored = firstFileToStored[j]; + if (iStored == invalidProcessIndex()) { + processBlockCacheIndices_[outerOffset_ + k].push_back(invalidCacheIndex()); + } else { + unsigned int oldCacheIndex = storedCacheIndices[storedIndex]; + ++storedIndex; + unsigned int newCacheIndex = invalidCacheIndex(); + if (oldCacheIndex != invalidCacheIndex()) { + newCacheIndex = oldCacheIndex + cacheIndexOffset_; + } + processBlockCacheIndices_[outerOffset_ + k].push_back(newCacheIndex); + } + } + } + cacheIndexVectorsPerFile_.push_back(nCacheIndexVectors); + fillEntriesFromPrimaryInput(std::move(newNEntries)); + } + + void ProcessBlockHelper::fillEntriesFromPrimaryInput(std::vector&& nEntries) { + unsigned int entriesThisFile = 0; + for (auto const& entries : nEntries) { + entriesThisFile += entries; + } + nEntries_.push_back(std::move(nEntries)); + cacheEntriesPerFile_.push_back(entriesThisFile); + cacheIndexOffset_ += entriesThisFile; + } + +} // namespace edm diff --git a/FWCore/Common/src/ProcessBlockHelperBase.cc b/FWCore/Common/src/ProcessBlockHelperBase.cc new file mode 100644 index 0000000000000..0d123adefe53a --- /dev/null +++ b/FWCore/Common/src/ProcessBlockHelperBase.cc @@ -0,0 +1,56 @@ +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "FWCore/Utilities/interface/BranchType.h" +#include "FWCore/Utilities/interface/ProductLabels.h" +#include "FWCore/Utilities/interface/TypeID.h" + +namespace edm { + + void ProcessBlockHelperBase::updateForNewProcess(ProductRegistry const& productRegistry, + std::string const& processName) { + // Add the current process at the end if there are any + // process blocks produced in the current process. + for (auto const& product : productRegistry.productList()) { + auto const& desc = product.second; + if (desc.branchType() == InProcess && desc.produced() && !desc.transient()) { + processesWithProcessBlockProducts_.emplace_back(processName); + addedProcesses_.emplace_back(processName); + return; + } + } + } + + std::string ProcessBlockHelperBase::selectProcess(ProductRegistry const& productRegistry, + ProductLabels const& productLabels, + TypeID const& typeID) const { + std::string processName(productLabels.process); + std::string selectedProcess; + + unsigned int bestPosition = 0; + for (auto const& prod : productRegistry.productList()) { + BranchDescription const& desc = prod.second; + if (desc.branchType() == InProcess && !desc.produced() && desc.present() && + desc.moduleLabel() == productLabels.module && desc.productInstanceName() == productLabels.productInstance && + desc.unwrappedTypeID() == typeID && (processName.empty() || processName == desc.processName())) { + // This code is to select the latest matching process + unsigned int position = 0; + bool found = false; + for (auto const& processFromHelper : processesWithProcessBlockProducts_) { + if (processFromHelper == desc.processName()) { + found = true; + break; + } + ++position; + } + if (found && position >= bestPosition) { + bestPosition = position; + selectedProcess = desc.processName(); + } + } + } + return selectedProcess; + } + +} // namespace edm diff --git a/FWCore/Common/src/SubProcessBlockHelper.cc b/FWCore/Common/src/SubProcessBlockHelper.cc new file mode 100644 index 0000000000000..157808e316201 --- /dev/null +++ b/FWCore/Common/src/SubProcessBlockHelper.cc @@ -0,0 +1,75 @@ +#include "FWCore/Common/interface/SubProcessBlockHelper.h" + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "FWCore/Utilities/interface/BranchType.h" + +#include +#include + +namespace edm { + + ProcessBlockHelperBase const* SubProcessBlockHelper::topProcessBlockHelper() const { return topProcessBlockHelper_; } + + std::vector const& SubProcessBlockHelper::topProcessesWithProcessBlockProducts() const { + return topProcessBlockHelper_->processesWithProcessBlockProducts(); + } + + unsigned int SubProcessBlockHelper::nProcessesInFirstFile() const { + return topProcessBlockHelper_->nProcessesInFirstFile(); + } + + std::vector> const& SubProcessBlockHelper::processBlockCacheIndices() const { + return topProcessBlockHelper_->processBlockCacheIndices(); + } + + std::vector> const& SubProcessBlockHelper::nEntries() const { + return topProcessBlockHelper_->nEntries(); + } + + std::vector const& SubProcessBlockHelper::cacheIndexVectorsPerFile() const { + return topProcessBlockHelper_->cacheIndexVectorsPerFile(); + } + + std::vector const& SubProcessBlockHelper::cacheEntriesPerFile() const { + return topProcessBlockHelper_->cacheEntriesPerFile(); + } + + unsigned int SubProcessBlockHelper::processBlockIndex( + std::string const& processName, EventToProcessBlockIndexes const& eventToProcessBlockIndexes) const { + return topProcessBlockHelper_->processBlockIndex(processName, eventToProcessBlockIndexes); + } + + unsigned int SubProcessBlockHelper::outerOffset() const { return topProcessBlockHelper_->outerOffset(); } + + void SubProcessBlockHelper::updateFromParentProcess(ProcessBlockHelperBase const& parentProcessBlockHelper, + ProductRegistry const& productRegistry) { + topProcessBlockHelper_ = parentProcessBlockHelper.topProcessBlockHelper(); + + // If a SubProcess keeps any ProcessBlock products from its parent process, then insert their + // process names. + assert(processesWithProcessBlockProducts().empty()); + for (auto const& processName : parentProcessBlockHelper.processesWithProcessBlockProducts()) { + for (auto const& item : productRegistry.productList()) { + BranchDescription const& desc = item.second; + if (desc.branchType() == InProcess && desc.present() && desc.processName() == processName) { + processesWithProcessBlockProducts().emplace_back(processName); + break; + } + } + } + + // Repeat for addedProcesses + assert(addedProcesses().empty()); + for (auto const& processName : parentProcessBlockHelper.addedProcesses()) { + for (auto const& item : productRegistry.productList()) { + BranchDescription const& desc = item.second; + if (desc.branchType() == InProcess && desc.present() && desc.processName() == processName) { + addedProcesses().emplace_back(processName); + break; + } + } + } + } + +} // namespace edm diff --git a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc new file mode 100644 index 0000000000000..483d35bf38511 --- /dev/null +++ b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc @@ -0,0 +1,36 @@ +//#define CATCH_CONFIG_MAIN +#include "catch.hpp" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" +#include "FWCore/Common/interface/SubProcessBlockHelper.h" + +#include +#include + +TEST_CASE("Test ProcessBlockHelpers", "[ProcessBlockHelpers]") { + const std::vector testNames = {{"HLT"}, {"RECO"}, {"TEST"}}; + const std::vector testNames2 = {{"MERGE"}, {"ANA"}, {"HARVEST"}}; + + SECTION("OutputProcessBlockHelper") { + edm::OutputProcessBlockHelper outputProcessBlockHelper; + + REQUIRE(edm::OutputProcessBlockHelper::invalidCacheIndex() == 0xffffffff); + } + + SECTION("ProcessBlockHelper") { + edm::ProcessBlockHelper processBlockHelper; + processBlockHelper.processesWithProcessBlockProducts() = testNames; + edm::ProcessBlockHelper const& constProcessBlockHelper = processBlockHelper; + REQUIRE(processBlockHelper.processesWithProcessBlockProducts() == testNames); + REQUIRE(constProcessBlockHelper.processesWithProcessBlockProducts() == testNames); + + processBlockHelper.addedProcesses() = testNames2; + REQUIRE(processBlockHelper.addedProcesses() == testNames2); + REQUIRE(constProcessBlockHelper.addedProcesses() == testNames2); + + REQUIRE(edm::ProcessBlockHelper::invalidCacheIndex() == 0xffffffff); + REQUIRE(edm::ProcessBlockHelper::invalidProcessIndex() == 0xffffffff); + } + + SECTION("SubProcessBlockHelper") { edm::SubProcessBlockHelper subProcessBlockHelper; } +} diff --git a/FWCore/Framework/interface/CacheHandle.h b/FWCore/Framework/interface/CacheHandle.h new file mode 100644 index 0000000000000..795064078a8a5 --- /dev/null +++ b/FWCore/Framework/interface/CacheHandle.h @@ -0,0 +1,35 @@ +#ifndef FWCore_Framework_CacheHandle_h +#define FWCore_Framework_CacheHandle_h + +/** \class edm::CacheHandle + +\author W. David Dagenhart, created 25 March, 2021 + +*/ + +#include "FWCore/Utilities/interface/Exception.h" + +namespace edm { + + template + class CacheHandle { + public: + CacheHandle() : data_(nullptr) {} + CacheHandle(T const* data) : data_(data) {} + + T const* get() const { + if (!isValid()) { + throw cms::Exception("InvalidCache") << "CacheHandle is invalid"; + } + return data_; + } + T const* operator->() const { return get(); } + T const& operator*() const { return *get(); } + + bool isValid() const { return data_ != nullptr; } + + private: + T const* data_; + }; +} // namespace edm +#endif diff --git a/FWCore/Framework/interface/EDAnalyzer.h b/FWCore/Framework/interface/EDAnalyzer.h index ba44bb31de8b8..423dd569c827d 100644 --- a/FWCore/Framework/interface/EDAnalyzer.h +++ b/FWCore/Framework/interface/EDAnalyzer.h @@ -74,6 +74,7 @@ namespace edm { bool doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDAnalyzer const*, ProductRegistry* reg); diff --git a/FWCore/Framework/interface/EDConsumerBase.h b/FWCore/Framework/interface/EDConsumerBase.h index 06e46742955bf..96d3a1d13a0a4 100644 --- a/FWCore/Framework/interface/EDConsumerBase.h +++ b/FWCore/Framework/interface/EDConsumerBase.h @@ -27,6 +27,8 @@ // user include files #include "DataFormats/Provenance/interface/BranchType.h" +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/ProductResolverIndexAndSkipBit.h" #include "FWCore/Framework/interface/EventSetupRecordKey.h" #include "FWCore/Framework/interface/HCTypeTag.h" @@ -50,10 +52,8 @@ // forward declarations namespace edm { - class ModuleDescription; class ModuleProcessName; class ProductResolverIndexHelper; - class ProductRegistry; class ConsumesCollector; template class EDConsumerBaseESAdaptor; @@ -102,6 +102,10 @@ namespace edm { // ---------- member functions --------------------------- void updateLookup(BranchType iBranchType, ProductResolverIndexHelper const&, bool iPrefetchMayGet); void updateLookup(eventsetup::ESRecordsToProxyIndices const&); + void selectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) { + doSelectInputProcessBlocks(productRegistry, processBlockHelperBase); + } typedef ProductLabels Labels; void labelsForToken(EDGetToken iToken, Labels& oLabels) const; @@ -253,6 +257,9 @@ namespace edm { void throwESConsumesInProcessBlock() const; edm::InputTag const& checkIfEmpty(edm::InputTag const& tag); + + virtual void doSelectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&); + // ---------- member data -------------------------------- struct TokenLookupInfo { diff --git a/FWCore/Framework/interface/EDFilter.h b/FWCore/Framework/interface/EDFilter.h index a84bc729c4b80..1a82c85252dd8 100644 --- a/FWCore/Framework/interface/EDFilter.h +++ b/FWCore/Framework/interface/EDFilter.h @@ -81,6 +81,7 @@ namespace edm { void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDFilter* module, ProductRegistry* reg) { diff --git a/FWCore/Framework/interface/EDProducer.h b/FWCore/Framework/interface/EDProducer.h index afe03a84a8b3a..ed0778b9a522b 100644 --- a/FWCore/Framework/interface/EDProducer.h +++ b/FWCore/Framework/interface/EDProducer.h @@ -77,6 +77,7 @@ namespace edm { void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDProducer* module, ProductRegistry* reg) { registerProducts(module, reg, moduleDescription_); diff --git a/FWCore/Framework/interface/Event.h b/FWCore/Framework/interface/Event.h index 31cb7d81aa98e..acb3401d95307 100644 --- a/FWCore/Framework/interface/Event.h +++ b/FWCore/Framework/interface/Event.h @@ -260,6 +260,10 @@ namespace edm { EDProductGetter const& productGetter() const; + unsigned int processBlockIndex(std::string const& processName) const { + return provRecorder_.processBlockIndex(processName); + } + private: //for testing friend class ::testEventGetRefBeforePut; diff --git a/FWCore/Framework/interface/EventForOutput.h b/FWCore/Framework/interface/EventForOutput.h index c5ae50834bed7..c4b0bb1cb7bd9 100644 --- a/FWCore/Framework/interface/EventForOutput.h +++ b/FWCore/Framework/interface/EventForOutput.h @@ -70,6 +70,8 @@ namespace edm { BranchListIndexes const& branchListIndexes() const; + EventToProcessBlockIndexes const& eventToProcessBlockIndexes() const; + EventSelectionIDVector const& eventSelectionIDs() const; ProductProvenanceRetriever const* productProvenanceRetrieverPtr() const; diff --git a/FWCore/Framework/interface/EventPrincipal.h b/FWCore/Framework/interface/EventPrincipal.h index bfc35e168b13c..d3a8bb0b6b642 100644 --- a/FWCore/Framework/interface/EventPrincipal.h +++ b/FWCore/Framework/interface/EventPrincipal.h @@ -17,6 +17,8 @@ is the DataBlock. #include "FWCore/Framework/interface/ProductProvenanceRetriever.h" #include "DataFormats/Provenance/interface/EventAuxiliary.h" #include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/Signal.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" @@ -57,7 +59,8 @@ namespace edm { ProcessConfiguration const& pc, HistoryAppender* historyAppender, unsigned int streamIndex = 0, - bool isForPrimaryProcess = true); + bool isForPrimaryProcess = true, + ProcessBlockHelperBase const* processBlockHelper = nullptr); ~EventPrincipal() override {} void fillEventPrincipal(EventAuxiliary const& aux, @@ -72,6 +75,7 @@ namespace edm { ProcessHistory const* processHistory, EventSelectionIDVector eventSelectionIDs, BranchListIndexes branchListIndexes, + EventToProcessBlockIndexes const&, ProductProvenanceRetriever const& provRetriever, DelayedReader* reader = nullptr, bool deepCopyRetriever = true); @@ -117,6 +121,8 @@ namespace edm { BranchListIndexes const& branchListIndexes() const; + EventToProcessBlockIndexes const& eventToProcessBlockIndexes() const; + Provenance const& getProvenance(ProductID const& pid) const; StableProvenance const& getStableProvenance(ProductID const& pid) const; @@ -151,6 +157,8 @@ namespace edm { using Base::getProvenance; using Base::getStableProvenance; + unsigned int processBlockIndex(std::string const& processName) const override; + private: BranchID pidToBid(ProductID const& pid) const; @@ -181,10 +189,13 @@ namespace edm { EventSelectionIDVector eventSelectionIDs_; std::shared_ptr branchIDListHelper_; + ProcessBlockHelperBase const* processBlockHelper_; std::shared_ptr thinnedAssociationsHelper_; BranchListIndexes branchListIndexes_; + EventToProcessBlockIndexes eventToProcessBlockIndexes_; + std::vector branchListIndexToProcessIndex_; StreamID streamID_; diff --git a/FWCore/Framework/interface/EventProcessor.h b/FWCore/Framework/interface/EventProcessor.h index 2713eae17f1f9..a6fc9b16daa25 100644 --- a/FWCore/Framework/interface/EventProcessor.h +++ b/FWCore/Framework/interface/EventProcessor.h @@ -12,6 +12,7 @@ configured in the user's main() function, and is set running. #include "DataFormats/Provenance/interface/RunID.h" #include "DataFormats/Provenance/interface/LuminosityBlockID.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputSource.h" #include "FWCore/Framework/interface/MergeableRunProductProcesses.h" @@ -32,6 +33,7 @@ configured in the user's main() function, and is set running. #include "FWCore/Concurrency/interface/LimitedTaskQueue.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" #include #include @@ -241,6 +243,7 @@ namespace edm { void handleEndLumiExceptions(std::exception_ptr const* iPtr, WaitingTaskHolder& holder); void globalEndLumiAsync(edm::WaitingTaskHolder iTask, std::shared_ptr iLumiStatus); void streamEndLumiAsync(edm::WaitingTaskHolder iTask, unsigned int iStreamIndex); + void readProcessBlock(ProcessBlockPrincipal&); std::pair readRun(); std::pair readAndMergeRun(); void readLuminosityBlock(LuminosityBlockProcessingStatus&); @@ -316,6 +319,7 @@ namespace edm { std::shared_ptr actReg_; // We do not use propagate_const because the registry itself is mutable. edm::propagate_const> preg_; edm::propagate_const> branchIDListHelper_; + edm::propagate_const> processBlockHelper_; edm::propagate_const> thinnedAssociationsHelper_; ServiceToken serviceToken_; edm::propagate_const> input_; @@ -337,7 +341,7 @@ namespace edm { std::vector subProcesses_; edm::propagate_const> historyAppender_; - edm::propagate_const> fb_; + edm::propagate_const> fb_; edm::propagate_const> looper_; //The atomic protects concurrent access of deferredExceptionPtr_ diff --git a/FWCore/Framework/interface/FileBlock.h b/FWCore/Framework/interface/FileBlock.h index 5a2e8e5cb9e65..b4a4a91631eda 100644 --- a/FWCore/Framework/interface/FileBlock.h +++ b/FWCore/Framework/interface/FileBlock.h @@ -14,6 +14,8 @@ class TTree; #include #include #include +#include +#include namespace edm { class BranchDescription; @@ -70,12 +72,14 @@ namespace edm { branchChildren_(new BranchChildren) {} FileBlock(FileFormatVersion const& version, - TTree const* ev, - TTree const* meta, - TTree const* lumi, - TTree const* lumiMeta, - TTree const* run, - TTree const* runMeta, + TTree* ev, + TTree* meta, + TTree* lumi, + TTree* lumiMeta, + TTree* run, + TTree* runMeta, + std::vector&& processBlockTrees, + std::vector&& processesWithProcessBlockTrees, int whyNotFastClonable, std::array const& hasNewlyDroppedBranch, std::string const& fileName, @@ -83,12 +87,14 @@ namespace edm { bool modifiedIDs, std::shared_ptr branchChildren) : fileFormatVersion_(version), - tree_(const_cast(ev)), - metaTree_(const_cast(meta)), - lumiTree_(const_cast(lumi)), - lumiMetaTree_(const_cast(lumiMeta)), - runTree_(const_cast(run)), - runMetaTree_(const_cast(runMeta)), + tree_(ev), + metaTree_(meta), + lumiTree_(lumi), + lumiMetaTree_(lumiMeta), + runTree_(run), + runMetaTree_(runMeta), + processBlockTrees_(std::move(processBlockTrees)), + processesWithProcessBlockTrees_(std::move(processesWithProcessBlockTrees)), whyNotFastClonable_(whyNotFastClonable), hasNewlyDroppedBranch_(hasNewlyDroppedBranch), fileName_(fileName), @@ -98,6 +104,15 @@ namespace edm { ~FileBlock() {} + void updateTTreePointers(TTree* ev, + TTree* meta, + TTree* lumi, + TTree* lumiMeta, + TTree* run, + TTree* runMeta, + std::vector&& processBlockTrees, + std::vector&& processesWithProcessBlockTrees); + FileFormatVersion const& fileFormatVersion() const { return fileFormatVersion_; } TTree* tree() const { return tree_; } TTree* metaTree() const { return metaTree_; } @@ -105,6 +120,10 @@ namespace edm { TTree* lumiMetaTree() const { return lumiMetaTree_; } TTree* runTree() const { return runTree_; } TTree* runMetaTree() const { return runMetaTree_; } + TTree* processBlockTree(std::string const& processName) const; + + std::vector const& processBlockTrees() const { return processBlockTrees_; } + std::vector const& processesWithProcessBlockTrees() const { return processesWithProcessBlockTrees_; } int whyNotFastClonable() const { return whyNotFastClonable_; } std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } @@ -114,7 +133,7 @@ namespace edm { void setNotFastClonable(WhyNotFastClonable const& why) { whyNotFastClonable_ |= why; } BranchChildren const& branchChildren() const { return *branchChildren_; } - void close() { runMetaTree_ = lumiMetaTree_ = metaTree_ = runTree_ = lumiTree_ = tree_ = nullptr; } + void close(); private: FileFormatVersion fileFormatVersion_; @@ -125,6 +144,8 @@ namespace edm { TTree* lumiMetaTree_; TTree* runTree_; TTree* runMetaTree_; + std::vector processBlockTrees_; + std::vector processesWithProcessBlockTrees_; int whyNotFastClonable_; std::array hasNewlyDroppedBranch_; std::string fileName_; diff --git a/FWCore/Framework/interface/Frameworkfwd.h b/FWCore/Framework/interface/Frameworkfwd.h index 19c76de6fcc8d..894fc5f56f8fa 100644 --- a/FWCore/Framework/interface/Frameworkfwd.h +++ b/FWCore/Framework/interface/Frameworkfwd.h @@ -40,6 +40,7 @@ namespace edm { class PrincipalCache; class PrincipalGetAdapter; class ProcessBlock; + class ProcessBlockForOutput; class ProcessBlockPrincipal; class ProcessBlockTransitionInfo; class ProcessNameSelector; diff --git a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h new file mode 100644 index 0000000000000..d9737af39b0f9 --- /dev/null +++ b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h @@ -0,0 +1,234 @@ +#ifndef FWCore_Framework_InputProcessBlockCacheImpl_h +#define FWCore_Framework_InputProcessBlockCacheImpl_h + +/** \class edm::impl::InputProcessBlockCacheImpl + +\author W. David Dagenhart, created 18 February, 2021 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" +#include "FWCore/Framework/interface/CacheHandle.h" +#include "FWCore/Framework/interface/EDConsumerBase.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/processBlockUtilities.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/ProductLabels.h" +#include "FWCore/Utilities/interface/TypeID.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace edm { + + class Event; + + namespace impl { + + template + constexpr std::size_t countTypeInParameterPack() { + return 0; + } + + template + constexpr std::size_t countTypeInParameterPack() { + return std::is_same::value ? 1 + countTypeInParameterPack() + : countTypeInParameterPack(); + } + + class InvalidCacheType {}; + + template + constexpr std::size_t indexInputProcessBlockCache() { + if constexpr (std::is_same::value) { + return 0; + } else { + if constexpr (sizeof...(Types) > 0) { + return 1 + indexInputProcessBlockCache(); + } else { + static_assert(sizeof...(Types) > 0, + "CacheType used with registerProcessBlockCacheFiller does not match any template parameters of " + "InputProcessBlockCache"); + return 0; + } + } + } + + struct TokenInfo { + public: + EDGetToken token_; + TypeID typeID_; + }; + + template + class CacheFiller { + public: + std::function(ProcessBlock const&, std::shared_ptr const&)> func_; + }; + + template + class InputProcessBlockCacheImpl { + public: + InputProcessBlockCacheImpl() = default; + InputProcessBlockCacheImpl(InputProcessBlockCacheImpl const&) = delete; + InputProcessBlockCacheImpl& operator=(InputProcessBlockCacheImpl const&) = delete; + + template + typename std::enable_if::type fillTuple(std::tuple...>&, + Event const&) const {} + + template + typename std::enable_if < + I::type fillTuple(std::tuple...>& cacheHandles, + Event const& event) const { + unsigned int index = eventProcessBlockIndex(event, processNames_[I]); + + // If the branch associated with the token was passed to registerProcessBlockCacheFiller + // was not in the input file, then the index will be invalid. Note the branch (including + // its process name) is selected using the first input file. Also note that even if + // the branch is present and this index is valid, the product might not be there. + // The functor in the CacheFiller must deal with that case. + if (index != ProcessBlockHelperBase::invalidCacheIndex()) { + std::get(cacheHandles) = CacheHandle(std::get(caches_.at(index).cacheTuple_).get()); + } + fillTuple(cacheHandles, event); + } + + std::tuple...> processBlockCaches(Event const& event) const { + std::tuple...> cacheHandles; + // processNames will be empty if and only if registerProcessBlockCacheFiller + // was never called by the module constructor + if (!processNames_.empty()) { + fillTuple<0>(cacheHandles, event); + } + return cacheHandles; + } + + void selectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase, + EDConsumerBase const& edConsumerBase) { + unsigned int i = 0; + for (auto const& tokenInfo : tokenInfos_) { + if (!tokenInfo.token_.isUninitialized()) { + ProductLabels productLabels; + edConsumerBase.labelsForToken(tokenInfo.token_, productLabels); + + processNames_[i] = processBlockHelperBase.selectProcess(productRegistry, productLabels, tokenInfo.typeID_); + } + ++i; + } + tokenInfos_.clear(); + } + + template + using CacheTypeT = typename std::tuple_element>::type; + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + registerProcessBlockCacheFiller, DataType, Func>(token, + std::forward(func)); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + static_assert(countTypeInParameterPack() == 1u, + "If registerProcessBlockCacheFiller is called with a type template parameter\n" + "then that type must appear exactly once in the template parameters of InputProcessBlockCache"); + + // Find the index into the parameter pack from the CacheType + constexpr unsigned int I = indexInputProcessBlockCache(); + + registerProcessBlockCacheFiller(token, std::forward(func)); + } + + // This gets used for stream type modules instead of registerProcessBlockCacheFiller + void copyProcessBlockCacheFiller(std::vector& tokenInfos, + std::tuple...>& functors) { + tokenInfos_ = std::move(tokenInfos); + functors_ = std::move(functors); + if (!tokenInfos_.empty()) { + processNames_.resize(sizeof...(CacheTypes)); + } + } + + // These are used to fill the CacheTuples + // One CacheFiller for each CacheType + + class CacheTuple { + public: + std::tuple...> cacheTuple_; + }; + + template + typename std::enable_if::type fillCache(ProcessBlock const&, + CacheTuple const&, + CacheTuple&) {} + + template + typename std::enable_if < I::type fillCache(ProcessBlock const& pb, + CacheTuple const& previousCacheTuple, + CacheTuple& cacheTuple) { + if (pb.processName() == processNames_[I]) { + auto const& previousSharedPtr = std::get(previousCacheTuple.cacheTuple_); + std::get(cacheTuple.cacheTuple_) = std::get(functors_).func_(pb, previousSharedPtr); + } + fillCache(pb, previousCacheTuple, cacheTuple); + } + + void accessInputProcessBlock(ProcessBlock const& pb) { + if (sizeof...(CacheTypes) > 0 && !processNames_.empty()) { + CacheTuple cacheTuple; + if (caches_.empty()) { + CacheTuple firstCacheTuple; + fillCache<0>(pb, firstCacheTuple, cacheTuple); + } else { + CacheTuple const& previousCacheTuple = caches_.back(); + fillCache<0>(pb, previousCacheTuple, cacheTuple); + } + caches_.push_back(std::move(cacheTuple)); + } + } + + void clearCaches() { caches_.clear(); } + auto cacheSize() const { return caches_.size(); } + + private: + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + static_assert(ICacheType < sizeof...(CacheTypes), "ICacheType out of range"); + processNames_.resize(sizeof...(CacheTypes)); + tokenInfos_.resize(sizeof...(CacheTypes)); + if (!tokenInfos_[ICacheType].token_.isUninitialized()) { + throw Exception(errors::LogicError) + << "registerProcessBlockCacheFiller should only be called once per cache type"; + } + + tokenInfos_[ICacheType] = TokenInfo{EDGetToken(token), TypeID(typeid(DataType))}; + + std::get(functors_).func_ = + std::function(ProcessBlock const&, std::shared_ptr const&)>( + std::forward(func)); + } + + // ------------ Data members -------------------- + + // This holds an entry per ProcessBlock + std::vector caches_; + + // The following 3 data members have one element for each CacheType + std::tuple...> functors_; + std::vector processNames_; + std::vector tokenInfos_; + }; + + } // namespace impl +} // namespace edm +#endif diff --git a/FWCore/Framework/interface/InputSource.h b/FWCore/Framework/interface/InputSource.h index be660e49bc4f6..80e4e3d9670ab 100644 --- a/FWCore/Framework/interface/InputSource.h +++ b/FWCore/Framework/interface/InputSource.h @@ -3,39 +3,14 @@ /*---------------------------------------------------------------------- -InputSource: Abstract interface for all input sources. Input -sources are responsible for creating an EventPrincipal, using data -controlled by the source, and external to the EventPrincipal itself. +InputSource: Abstract interface for all input sources. -The InputSource is also responsible for dealing with the "process -name list" contained within the EventPrincipal. Each InputSource has -to know what "process" (HLT, PROD, USER, USER1, etc.) the program is -part of. The InputSource is repsonsible for pushing this process name -onto the end of the process name list. +Some examples of InputSource subclasses are: -For now, we specify this process name to the constructor of the -InputSource. This should be improved. - - Some questions about this remain: - - 1. What should happen if we "rerun" a process? i.e., if "USER1" is - already last in our input file, and we run again a job which claims - to be "USER1", what should happen? For now, we just quietly add - this to the history. - - 2. Do we need to detect a problem with a history like: - HLT PROD USER1 PROD - or is it up to the user not to do something silly? Right now, there - is no protection against such sillyness. - -Some examples of InputSource subclasses may be: - - 1) EmptySource: creates EventPrincipals which contain no EDProducts. - 2) PoolSource: creates EventPrincipals which "contain" the data - read from a EDM/ROOT file. This source should provide for delayed loading - of data, thus the quotation marks around contain. - 3) DAQSource: creats EventPrincipals which contain raw data, as - delivered by the L1 trigger and event builder. + 1) PoolSource: Handles things related to reading from an EDM/ROOT file. + This source provides for delayed loading of data. + 2) EmptySource: Handles similar tasks for the case where there is no + data in the input. ----------------------------------------------------------------------*/ @@ -45,6 +20,7 @@ Some examples of InputSource subclasses may be: #include "DataFormats/Provenance/interface/RunAuxiliary.h" #include "DataFormats/Provenance/interface/RunID.h" #include "DataFormats/Provenance/interface/Timestamp.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/ProcessingController.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" @@ -120,11 +96,17 @@ namespace edm { /// Read next luminosity block (same as a prior lumi) void readAndMergeLumi(LuminosityBlockPrincipal& lbp); + /// Fill the ProcessBlockHelper with info for the current file + void fillProcessBlockHelper(); + + /// Next process block, return false if there is none, sets the processName in the principal + bool nextProcessBlock(ProcessBlockPrincipal&); + /// Read next process block - bool readProcessBlock(); + void readProcessBlock(ProcessBlockPrincipal&); /// Read next file - std::unique_ptr readFile(); + std::shared_ptr readFile(); /// close current file void closeFile(FileBlock*, bool cleaningUpAfterException); @@ -164,6 +146,12 @@ namespace edm { } std::shared_ptr& branchIDListHelper() { return get_underlying_safe(branchIDListHelper_); } + /// Accessors for processBlockHelper + std::shared_ptr processBlockHelper() const { + return get_underlying_safe(processBlockHelper_); + } + std::shared_ptr& processBlockHelper() { return get_underlying_safe(processBlockHelper_); } + /// Accessors for thinnedAssociationsHelper std::shared_ptr thinnedAssociationsHelper() const { return get_underlying_safe(thinnedAssociationsHelper_); @@ -287,6 +275,19 @@ namespace edm { RunIndex index_; }; + class ProcessBlockSourceSentry { + public: + ProcessBlockSourceSentry(InputSource const&, std::string const&); + ~ProcessBlockSourceSentry(); + + ProcessBlockSourceSentry(ProcessBlockSourceSentry const&) = delete; // Disallow copying and moving + ProcessBlockSourceSentry& operator=(ProcessBlockSourceSentry const&) = delete; // Disallow copying and moving + + private: + InputSource const& source_; + std::string const& processName_; + }; + class FileOpenSentry { public: typedef signalslot::Signal Sig; @@ -389,11 +390,14 @@ namespace edm { ItemType nextItemType_(); virtual std::shared_ptr readRunAuxiliary_() = 0; virtual std::shared_ptr readLuminosityBlockAuxiliary_() = 0; + virtual void fillProcessBlockHelper_(); + virtual bool nextProcessBlock_(ProcessBlockPrincipal&); + virtual void readProcessBlock_(ProcessBlockPrincipal&); virtual void readRun_(RunPrincipal& runPrincipal); virtual void readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal); virtual void readEvent_(EventPrincipal& eventPrincipal) = 0; virtual bool readIt(EventID const& id, EventPrincipal& eventPrincipal, StreamContext& streamContext); - virtual std::unique_ptr readFile_(); + virtual std::shared_ptr readFile_(); virtual void closeFile_() {} virtual bool goToEvent_(EventID const& eventID); virtual void setRun(RunNumber_t r); @@ -420,6 +424,7 @@ namespace edm { edm::propagate_const> productRegistry_; edm::propagate_const> processHistoryRegistry_; edm::propagate_const> branchIDListHelper_; + edm::propagate_const> processBlockHelper_; edm::propagate_const> thinnedAssociationsHelper_; std::string processGUID_; Timestamp time_; diff --git a/FWCore/Framework/interface/InputSourceDescription.h b/FWCore/Framework/interface/InputSourceDescription.h index ca8476c98b81d..5aafcbc743b22 100644 --- a/FWCore/Framework/interface/InputSourceDescription.h +++ b/FWCore/Framework/interface/InputSourceDescription.h @@ -8,6 +8,7 @@ input source that does not come in through the ParameterSet ----------------------------------------------------------------------*/ #include #include "DataFormats/Provenance/interface/ModuleDescription.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/src/PreallocationConfiguration.h" namespace edm { @@ -29,6 +30,7 @@ namespace edm { InputSourceDescription(ModuleDescription const& md, std::shared_ptr preg, std::shared_ptr branchIDListHelper, + std::shared_ptr const& processBlockHelper, std::shared_ptr thinnedAssociationsHelper, std::shared_ptr areg, int maxEvents, @@ -38,6 +40,7 @@ namespace edm { : moduleDescription_(md), productRegistry_(preg), branchIDListHelper_(branchIDListHelper), + processBlockHelper_(processBlockHelper), thinnedAssociationsHelper_(thinnedAssociationsHelper), actReg_(areg), maxEvents_(maxEvents), @@ -48,6 +51,7 @@ namespace edm { ModuleDescription moduleDescription_; std::shared_ptr productRegistry_; std::shared_ptr branchIDListHelper_; + std::shared_ptr processBlockHelper_; std::shared_ptr thinnedAssociationsHelper_; std::shared_ptr actReg_; // We do not use propagate_const because the registry itself is mutable. int maxEvents_; diff --git a/FWCore/Framework/interface/OccurrenceForOutput.h b/FWCore/Framework/interface/OccurrenceForOutput.h index 107e048b5c881..104e38e7904fc 100644 --- a/FWCore/Framework/interface/OccurrenceForOutput.h +++ b/FWCore/Framework/interface/OccurrenceForOutput.h @@ -6,7 +6,7 @@ // Package: Framework // Class : OccurrenceForOutput // -/**\class OccurrenceForOutput OccurrenceForOutputForOutput.h FWCore/Framework/interface/OccurrenceForOutputForOutput.h +/**\class edm::OccurrenceForOutput */ /*---------------------------------------------------------------------- diff --git a/FWCore/Framework/interface/Principal.h b/FWCore/Framework/interface/Principal.h index 03f7b6b995954..9b1b6cfc38c9a 100644 --- a/FWCore/Framework/interface/Principal.h +++ b/FWCore/Framework/interface/Principal.h @@ -198,6 +198,8 @@ namespace edm { ConstProductResolverPtr getProductResolverByIndex(ProductResolverIndex const& oid) const; + virtual unsigned int processBlockIndex(std::string const& processName) const; + protected: // ----- Add a new ProductResolver // *this takes ownership of the ProductResolver, which in turn owns its diff --git a/FWCore/Framework/interface/PrincipalGetAdapter.h b/FWCore/Framework/interface/PrincipalGetAdapter.h index 4ee57124afdff..54a8986898b34 100644 --- a/FWCore/Framework/interface/PrincipalGetAdapter.h +++ b/FWCore/Framework/interface/PrincipalGetAdapter.h @@ -218,6 +218,8 @@ namespace edm { void labelsForToken(EDGetToken const& iToken, ProductLabels& oLabels) const; + unsigned int processBlockIndex(std::string const& processName) const; + private: // Is this an Event, a LuminosityBlock, or a Run. BranchType const& branchType() const; diff --git a/FWCore/Framework/interface/ProcessBlock.h b/FWCore/Framework/interface/ProcessBlock.h index a231ad7bd4c70..9729a07fdef90 100644 --- a/FWCore/Framework/interface/ProcessBlock.h +++ b/FWCore/Framework/interface/ProcessBlock.h @@ -77,6 +77,8 @@ namespace edm { ModuleCallingContext const* moduleCallingContext() const { return moduleCallingContext_; } + std::string const& processName() const; + private: ProcessBlockPrincipal const& processBlockPrincipal() const; diff --git a/FWCore/Framework/interface/ProcessBlockForOutput.h b/FWCore/Framework/interface/ProcessBlockForOutput.h new file mode 100644 index 0000000000000..d2a5d0fda76a3 --- /dev/null +++ b/FWCore/Framework/interface/ProcessBlockForOutput.h @@ -0,0 +1,41 @@ +#ifndef FWCore_Framework_ProcessBlockForOutput_h +#define FWCore_Framework_ProcessBlockForOutput_h + +// -*- C++ -*- +// +// Package: Framework +// Class : ProcessBlockForOutput +// +/**\class edm::ProcessBlockForOutput + +Description: This is the primary interface for output modules +writing ProcessBlock products + +\author W. David Dagenhart, created 29 October 2020 + +*/ + +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Framework/interface/OccurrenceForOutput.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h" + +#include + +namespace edm { + class ProcessBlockPrincipal; + + class ProcessBlockForOutput : public OccurrenceForOutput { + public: + ProcessBlockForOutput(ProcessBlockPrincipal const&, + ModuleDescription const&, + ModuleCallingContext const*, + bool isAtEnd); + ~ProcessBlockForOutput() override; + + std::string const& processName() const { return *processName_; } + + private: + std::string const* processName_; + }; +} // namespace edm +#endif diff --git a/FWCore/Framework/interface/Schedule.h b/FWCore/Framework/interface/Schedule.h index 81d655368f5fa..ad3204e8246a0 100644 --- a/FWCore/Framework/interface/Schedule.h +++ b/FWCore/Framework/interface/Schedule.h @@ -58,6 +58,7 @@ */ #include "DataFormats/Provenance/interface/ModuleDescription.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/ExceptionActions.h" #include "FWCore/Framework/interface/ExceptionHelpers.h" #include "FWCore/Framework/interface/Frameworkfwd.h" @@ -130,6 +131,7 @@ namespace edm { service::TriggerNamesService const& tns, ProductRegistry& pregistry, BranchIDListHelper& branchIDListHelper, + ProcessBlockHelperBase&, ThinnedAssociationsHelper& thinnedAssociationsHelper, SubProcessParentageHelper const* subProcessParentageHelper, ExceptionToActionTable const& actions, @@ -157,7 +159,7 @@ namespace edm { ServiceToken const& token, bool cleaningUpAfterException = false); - void beginJob(ProductRegistry const&, eventsetup::ESRecordsToProxyIndices const&); + void beginJob(ProductRegistry const&, eventsetup::ESRecordsToProxyIndices const&, ProcessBlockHelperBase const&); void endJob(ExceptionCollector& collector); void beginStream(unsigned int); diff --git a/FWCore/Framework/interface/ScheduleItems.h b/FWCore/Framework/interface/ScheduleItems.h index afba98427cd1d..e83f1f14178ed 100644 --- a/FWCore/Framework/interface/ScheduleItems.h +++ b/FWCore/Framework/interface/ScheduleItems.h @@ -1,9 +1,11 @@ #ifndef FWCore_Framework_ScheduleItems_h #define FWCore_Framework_ScheduleItems_h +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/ServiceRegistry/interface/ServiceLegacy.h" #include "FWCore/ServiceRegistry/interface/ServiceToken.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" #include #include @@ -28,7 +30,10 @@ namespace edm { struct ScheduleItems { ScheduleItems(); - ScheduleItems(ProductRegistry const& preg, SubProcess const& om); + ScheduleItems(ProductRegistry const& preg, + SubProcess const& om, + SubProcessBlockHelper& subProcessBlockHelper, + ProcessBlockHelperBase const& parentProcessBlockHelper); ScheduleItems(ScheduleItems const&) = delete; // Disallow copying and moving ScheduleItems& operator=(ScheduleItems const&) = delete; // Disallow copying and moving @@ -46,7 +51,8 @@ namespace edm { std::unique_ptr initSchedule(ParameterSet& parameterSet, bool hasSubprocesses, PreallocationConfiguration const& iAllocConfig, - ProcessContext const*); + ProcessContext const*, + ProcessBlockHelperBase& processBlockHelper); std::shared_ptr preg() const { return get_underlying_safe(preg_); } std::shared_ptr& preg() { return get_underlying_safe(preg_); } diff --git a/FWCore/Framework/interface/WorkerManager.h b/FWCore/Framework/interface/WorkerManager.h index 4135b42cbd967..01b212849364e 100644 --- a/FWCore/Framework/interface/WorkerManager.h +++ b/FWCore/Framework/interface/WorkerManager.h @@ -5,6 +5,7 @@ */ +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/ExceptionHelpers.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/UnscheduledCallProducer.h" @@ -72,7 +73,9 @@ namespace edm { void setupResolvers(Principal& principal); void setupOnDemandSystem(EventTransitionInfo const&); - void beginJob(ProductRegistry const& iRegistry, eventsetup::ESRecordsToProxyIndices const&); + void beginJob(ProductRegistry const& iRegistry, + eventsetup::ESRecordsToProxyIndices const&, + ProcessBlockHelperBase const&); void endJob(); void endJob(ExceptionCollector& collector); diff --git a/FWCore/Framework/interface/global/EDAnalyzerBase.h b/FWCore/Framework/interface/global/EDAnalyzerBase.h index b5bbda7d14fd4..162b94cb71ed5 100644 --- a/FWCore/Framework/interface/global/EDAnalyzerBase.h +++ b/FWCore/Framework/interface/global/EDAnalyzerBase.h @@ -98,9 +98,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDAnalyzerBase* module, ProductRegistry* reg); @@ -135,6 +135,8 @@ namespace edm { virtual void doEndLuminosityBlockSummary_(LuminosityBlock const& lb, EventSetup const& c); virtual void doEndLuminosityBlock_(LuminosityBlock const& lb, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); + bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/global/EDFilterBase.h b/FWCore/Framework/interface/global/EDFilterBase.h index 5a4952992de34..5b68eb730a739 100644 --- a/FWCore/Framework/interface/global/EDFilterBase.h +++ b/FWCore/Framework/interface/global/EDFilterBase.h @@ -101,9 +101,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDFilterBase* module, ProductRegistry* reg) { @@ -147,6 +147,7 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); virtual bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/global/EDProducerBase.h b/FWCore/Framework/interface/global/EDProducerBase.h index 2869125c864cb..f1eeb792e9e7e 100644 --- a/FWCore/Framework/interface/global/EDProducerBase.h +++ b/FWCore/Framework/interface/global/EDProducerBase.h @@ -99,9 +99,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDProducerBase* module, ProductRegistry* reg) { @@ -150,6 +150,7 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } virtual bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/global/OutputModuleBase.h b/FWCore/Framework/interface/global/OutputModuleBase.h index 26e51f244545a..02b98f720fe07 100644 --- a/FWCore/Framework/interface/global/OutputModuleBase.h +++ b/FWCore/Framework/interface/global/OutputModuleBase.h @@ -32,6 +32,8 @@ #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/ProductSelectorRules.h" @@ -87,7 +89,7 @@ namespace edm { bool selected(BranchDescription const& desc) const; - void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&); + void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&); std::string const& processName() const { return process_name_; } SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; } std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } @@ -101,6 +103,8 @@ namespace edm { BranchIDLists const* branchIDLists() const; + OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; } + ThinnedAssociationsHelper const* thinnedAssociationsHelper() const; const ModuleDescription& moduleDescription() const { return moduleDescription_; } @@ -194,18 +198,21 @@ namespace edm { edm::propagate_const> thinnedAssociationsHelper_; std::map keepAssociation_; + OutputProcessBlockHelper outputProcessBlockHelper_; + //------------------------------------------------------------------ // private member functions //------------------------------------------------------------------ void updateBranchIDListsWithKeptAliases(); - void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {} + void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*); void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*); void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*); void doOpenFile(FileBlock const& fb); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} std::string workerType() const { return "WorkerT"; } @@ -231,6 +238,7 @@ namespace edm { virtual void endJob() {} virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0; virtual void writeRun(RunForOutput const&) = 0; + virtual void writeProcessBlock(ProcessBlockForOutput const&) {} virtual void openFile(FileBlock const&) {} virtual bool isFileOpen() const { return true; } diff --git a/FWCore/Framework/interface/global/analyzerAbilityToImplementor.h b/FWCore/Framework/interface/global/analyzerAbilityToImplementor.h index df0791109aea4..b29311d6f0652 100644 --- a/FWCore/Framework/interface/global/analyzerAbilityToImplementor.h +++ b/FWCore/Framework/interface/global/analyzerAbilityToImplementor.h @@ -34,37 +34,37 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::global::impl::StreamCacheHolder Type; + using Type = edm::global::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::global::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::global::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunCacheHolder Type; + using Type = edm::global::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunSummaryCacheHolder Type; + using Type = edm::global::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::WatchProcessBlock Type; + using Type = edm::global::impl::WatchProcessBlock; }; } // namespace analyzer } // namespace global diff --git a/FWCore/Framework/interface/global/filterAbilityToImplementor.h b/FWCore/Framework/interface/global/filterAbilityToImplementor.h index 2669601b50384..8867b849489dc 100644 --- a/FWCore/Framework/interface/global/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/global/filterAbilityToImplementor.h @@ -34,96 +34,96 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::global::impl::StreamCacheHolder Type; + using Type = edm::global::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::global::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::global::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunCacheHolder Type; + using Type = edm::global::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunSummaryCacheHolder Type; + using Type = edm::global::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::WatchProcessBlock Type; + using Type = edm::global::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginProcessBlockProducer Type; + using Type = edm::global::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndProcessBlockProducer Type; + using Type = edm::global::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginRunProducer Type; + using Type = edm::global::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndRunProducer Type; + using Type = edm::global::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginLuminosityBlockProducer Type; + using Type = edm::global::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndLuminosityBlockProducer Type; + using Type = edm::global::impl::EndLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::ExternalWork Type; + using Type = edm::global::impl::ExternalWork; }; template struct SpecializeAbilityToImplementor { - typedef typename AbilityToImplementor::Type Type; + using Type = typename AbilityToImplementor::Type; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::global::impl::EndRunSummaryProducer Type; + using Type = typename edm::global::impl::EndRunSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::global::impl::EmptyType Type; + using Type = edm::global::impl::EmptyType; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::global::impl::EndLuminosityBlockSummaryProducer Type; + using Type = typename edm::global::impl::EndLuminosityBlockSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::global::impl::EmptyType Type; + using Type = edm::global::impl::EmptyType; }; } // namespace filter } // namespace global diff --git a/FWCore/Framework/interface/global/implementors.h b/FWCore/Framework/interface/global/implementors.h index 9d39d4981bca3..220b31e55b935 100644 --- a/FWCore/Framework/interface/global/implementors.h +++ b/FWCore/Framework/interface/global/implementors.h @@ -19,12 +19,21 @@ // // system include files +#include #include #include +#include +#include +#include +#include // user include files +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" @@ -87,7 +96,7 @@ namespace edm { std::vector caches_; }; - template + template class InputProcessBlockCacheHolder : public virtual T { public: InputProcessBlockCacheHolder() = default; @@ -95,21 +104,43 @@ namespace edm { InputProcessBlockCacheHolder& operator=(InputProcessBlockCacheHolder const&) = delete; ~InputProcessBlockCacheHolder() override {} - protected: - // Not implemented yet - // const C* inputProcessBlockCache(ProcessBlockIndex index) const { return caches_.at(index).get(); } + std::tuple...> processBlockCaches(Event const& event) const { + return cacheImpl_.processBlockCaches(event); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + // This is intended for use by Framework unit tests only + unsigned int cacheSize() const { return cacheImpl_.cacheSize(); } private: - // Not yet fully implemented, will never get called - // THINK ABOUT HOW TO CLEAR CACHES!!! + void doSelectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) final { + cacheImpl_.selectInputProcessBlocks(productRegistry, processBlockHelperBase, *this); + } + void doAccessInputProcessBlock_(ProcessBlock const& pb) final { - caches_.push_back(accessInputProcessBlock(pb)); + cacheImpl_.accessInputProcessBlock(pb); + accessInputProcessBlock(pb); } - // Not yet fully implemented, will never get called - virtual std::shared_ptr accessInputProcessBlock(ProcessBlock const&) const = 0; + // Alternate method to access ProcessBlocks without using the caches + // Mostly intended for unit testing, but might have other uses... + virtual void accessInputProcessBlock(ProcessBlock const&) {} + + void clearInputProcessBlockCaches() final { cacheImpl_.clearCaches(); } - std::vector> caches_; + edm::impl::InputProcessBlockCacheImpl cacheImpl_; }; template @@ -255,8 +286,8 @@ namespace edm { void doBeginProcessBlock_(ProcessBlock const&) final; void doEndProcessBlock_(ProcessBlock const&) final; - virtual void beginProcessBlock(ProcessBlock const&) const {} - virtual void endProcessBlock(ProcessBlock const&) const {} + virtual void beginProcessBlock(ProcessBlock const&) {} + virtual void endProcessBlock(ProcessBlock const&) {} }; template @@ -270,7 +301,7 @@ namespace edm { private: void doBeginProcessBlockProduce_(ProcessBlock&) final; - virtual void beginProcessBlockProduce(edm::ProcessBlock&) const = 0; + virtual void beginProcessBlockProduce(edm::ProcessBlock&) = 0; }; template @@ -284,7 +315,7 @@ namespace edm { private: void doEndProcessBlockProduce_(ProcessBlock&) final; - virtual void endProcessBlockProduce(edm::ProcessBlock&) const = 0; + virtual void endProcessBlockProduce(edm::ProcessBlock&) = 0; }; template diff --git a/FWCore/Framework/interface/global/producerAbilityToImplementor.h b/FWCore/Framework/interface/global/producerAbilityToImplementor.h index f25ae4d7f8b1b..54085f462b1d0 100644 --- a/FWCore/Framework/interface/global/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/global/producerAbilityToImplementor.h @@ -34,102 +34,102 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::global::impl::StreamCacheHolder Type; + using Type = edm::global::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::global::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::global::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunCacheHolder Type; + using Type = edm::global::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::RunSummaryCacheHolder Type; + using Type = edm::global::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::global::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::global::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::WatchProcessBlock Type; + using Type = edm::global::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginProcessBlockProducer Type; + using Type = edm::global::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndProcessBlockProducer Type; + using Type = edm::global::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginRunProducer Type; + using Type = edm::global::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndRunProducer Type; + using Type = edm::global::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::BeginLuminosityBlockProducer Type; + using Type = edm::global::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::EndLuminosityBlockProducer Type; + using Type = edm::global::impl::EndLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::ExternalWork Type; + using Type = edm::global::impl::ExternalWork; }; template <> struct AbilityToImplementor { - typedef edm::global::impl::Accumulator Type; + using Type = edm::global::impl::Accumulator; }; template struct SpecializeAbilityToImplementor { - typedef typename AbilityToImplementor::Type Type; + using Type = typename AbilityToImplementor::Type; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::global::impl::EndRunSummaryProducer Type; + using Type = typename edm::global::impl::EndRunSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::global::impl::EmptyType Type; + using Type = edm::global::impl::EmptyType; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::global::impl::EndLuminosityBlockSummaryProducer Type; + using Type = typename edm::global::impl::EndLuminosityBlockSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::global::impl::EmptyType Type; + using Type = edm::global::impl::EmptyType; }; } // namespace producer } // namespace global diff --git a/FWCore/Framework/interface/limited/EDAnalyzerBase.h b/FWCore/Framework/interface/limited/EDAnalyzerBase.h index 648c4f9798c4d..4a83c1a43ca02 100644 --- a/FWCore/Framework/interface/limited/EDAnalyzerBase.h +++ b/FWCore/Framework/interface/limited/EDAnalyzerBase.h @@ -103,9 +103,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDAnalyzerBase* module, ProductRegistry* reg); @@ -140,6 +140,8 @@ namespace edm { virtual void doEndLuminosityBlockSummary_(LuminosityBlock const& lb, EventSetup const& c); virtual void doEndLuminosityBlock_(LuminosityBlock const& lb, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); + bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/limited/EDFilterBase.h b/FWCore/Framework/interface/limited/EDFilterBase.h index 67d8e47c94c60..f0d8c54c710f5 100644 --- a/FWCore/Framework/interface/limited/EDFilterBase.h +++ b/FWCore/Framework/interface/limited/EDFilterBase.h @@ -101,9 +101,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDFilterBase* module, ProductRegistry* reg) { @@ -147,6 +147,8 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); + bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/limited/EDProducerBase.h b/FWCore/Framework/interface/limited/EDProducerBase.h index 9538e2a03fbe8..7c63a4f63da7a 100644 --- a/FWCore/Framework/interface/limited/EDProducerBase.h +++ b/FWCore/Framework/interface/limited/EDProducerBase.h @@ -100,8 +100,9 @@ namespace edm { void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDProducerBase* module, ProductRegistry* reg) { @@ -150,6 +151,7 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/limited/OutputModuleBase.h b/FWCore/Framework/interface/limited/OutputModuleBase.h index 27859a7451ddd..1fa70f257003b 100644 --- a/FWCore/Framework/interface/limited/OutputModuleBase.h +++ b/FWCore/Framework/interface/limited/OutputModuleBase.h @@ -32,6 +32,8 @@ #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/ProductSelectorRules.h" @@ -88,7 +90,7 @@ namespace edm { bool selected(BranchDescription const& desc) const; - void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&); + void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&); std::string const& processName() const { return process_name_; } SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; } std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } @@ -102,6 +104,8 @@ namespace edm { BranchIDLists const* branchIDLists() const; + OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; } + ThinnedAssociationsHelper const* thinnedAssociationsHelper() const; const ModuleDescription& moduleDescription() const { return moduleDescription_; } @@ -197,6 +201,9 @@ namespace edm { edm::propagate_const> thinnedAssociationsHelper_; std::map keepAssociation_; + + OutputProcessBlockHelper outputProcessBlockHelper_; + LimitedTaskQueue queue_; //------------------------------------------------------------------ @@ -205,12 +212,13 @@ namespace edm { void updateBranchIDListsWithKeptAliases(); - void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {} + void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*); void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*); void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*); void doOpenFile(FileBlock const& fb); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} std::string workerType() const { return "WorkerT"; } @@ -236,6 +244,7 @@ namespace edm { virtual void endJob() {} virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0; virtual void writeRun(RunForOutput const&) = 0; + virtual void writeProcessBlock(ProcessBlockForOutput const&) {} virtual void openFile(FileBlock const&) const {} virtual bool isFileOpen() const { return true; } diff --git a/FWCore/Framework/interface/limited/analyzerAbilityToImplementor.h b/FWCore/Framework/interface/limited/analyzerAbilityToImplementor.h index c1892ca242836..08d4f54d612ce 100644 --- a/FWCore/Framework/interface/limited/analyzerAbilityToImplementor.h +++ b/FWCore/Framework/interface/limited/analyzerAbilityToImplementor.h @@ -34,37 +34,37 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::limited::impl::StreamCacheHolder Type; + using Type = edm::limited::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::limited::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::limited::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunCacheHolder Type; + using Type = edm::limited::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunSummaryCacheHolder Type; + using Type = edm::limited::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::WatchProcessBlock Type; + using Type = edm::limited::impl::WatchProcessBlock; }; } // namespace analyzer } // namespace limited diff --git a/FWCore/Framework/interface/limited/filterAbilityToImplementor.h b/FWCore/Framework/interface/limited/filterAbilityToImplementor.h index 047155773044f..8d3ef2fef6f3c 100644 --- a/FWCore/Framework/interface/limited/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/limited/filterAbilityToImplementor.h @@ -34,92 +34,92 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::limited::impl::StreamCacheHolder Type; + using Type = edm::limited::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::limited::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::limited::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunCacheHolder Type; + using Type = edm::limited::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunSummaryCacheHolder Type; + using Type = edm::limited::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::WatchProcessBlock Type; + using Type = edm::limited::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginProcessBlockProducer Type; + using Type = edm::limited::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndProcessBlockProducer Type; + using Type = edm::limited::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginRunProducer Type; + using Type = edm::limited::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndRunProducer Type; + using Type = edm::limited::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginLuminosityBlockProducer Type; + using Type = edm::limited::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndLuminosityBlockProducer Type; + using Type = edm::limited::impl::EndLuminosityBlockProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename AbilityToImplementor::Type Type; + using Type = typename AbilityToImplementor::Type; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::limited::impl::EndRunSummaryProducer Type; + using Type = typename edm::limited::impl::EndRunSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::limited::impl::EmptyType Type; + using Type = edm::limited::impl::EmptyType; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::limited::impl::EndLuminosityBlockSummaryProducer Type; + using Type = typename edm::limited::impl::EndLuminosityBlockSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::limited::impl::EmptyType Type; + using Type = edm::limited::impl::EmptyType; }; } // namespace filter } // namespace limited diff --git a/FWCore/Framework/interface/limited/implementors.h b/FWCore/Framework/interface/limited/implementors.h index 701fb1234f53b..e69659e4cd452 100644 --- a/FWCore/Framework/interface/limited/implementors.h +++ b/FWCore/Framework/interface/limited/implementors.h @@ -21,10 +21,18 @@ // system include files #include #include +#include +#include +#include +#include // user include files +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" @@ -88,7 +96,7 @@ namespace edm { std::vector caches_; }; - template + template class InputProcessBlockCacheHolder : public virtual T { public: InputProcessBlockCacheHolder(edm::ParameterSet const& iPSet) : T(iPSet) {} @@ -96,20 +104,43 @@ namespace edm { InputProcessBlockCacheHolder& operator=(InputProcessBlockCacheHolder const&) = delete; ~InputProcessBlockCacheHolder() noexcept(false) override {} - protected: - // Not implemented yet - // const C* inputProcessBlockCache(ProcessBlockIndex index) const { return caches_.at(index).get(); } + std::tuple...> processBlockCaches(Event const& event) const { + return cacheImpl_.processBlockCaches(event); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + // This is intended for use by Framework unit tests only + unsigned int cacheSize() const { return cacheImpl_.cacheSize(); } private: - // Not yet fully implemented, will never get called + void doSelectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) final { + cacheImpl_.selectInputProcessBlocks(productRegistry, processBlockHelperBase, *this); + } + void doAccessInputProcessBlock_(ProcessBlock const& pb) final { - caches_.push_back(accessInputProcessBlock(pb)); + cacheImpl_.accessInputProcessBlock(pb); + accessInputProcessBlock(pb); } - // Not yet fully implemented, will never get called - virtual std::shared_ptr accessInputProcessBlock(ProcessBlock const&) const = 0; + // Alternate method to access ProcessBlocks without using the caches + // Mostly intended for unit testing, but might have other uses... + virtual void accessInputProcessBlock(ProcessBlock const&) {} + + void clearInputProcessBlockCaches() final { cacheImpl_.clearCaches(); } - std::vector> caches_; + edm::impl::InputProcessBlockCacheImpl cacheImpl_; }; template @@ -255,8 +286,8 @@ namespace edm { void doBeginProcessBlock_(ProcessBlock const&) final; void doEndProcessBlock_(ProcessBlock const&) final; - virtual void beginProcessBlock(ProcessBlock const&) const {} - virtual void endProcessBlock(ProcessBlock const&) const {} + virtual void beginProcessBlock(ProcessBlock const&) {} + virtual void endProcessBlock(ProcessBlock const&) {} }; template @@ -270,7 +301,7 @@ namespace edm { private: void doBeginProcessBlockProduce_(ProcessBlock&) final; - virtual void beginProcessBlockProduce(edm::ProcessBlock&) const = 0; + virtual void beginProcessBlockProduce(edm::ProcessBlock&) = 0; }; template @@ -284,7 +315,7 @@ namespace edm { private: void doEndProcessBlockProduce_(ProcessBlock&) final; - virtual void endProcessBlockProduce(edm::ProcessBlock&) const = 0; + virtual void endProcessBlockProduce(edm::ProcessBlock&) = 0; }; template diff --git a/FWCore/Framework/interface/limited/producerAbilityToImplementor.h b/FWCore/Framework/interface/limited/producerAbilityToImplementor.h index 89db9d304e5c3..6f179e2055713 100644 --- a/FWCore/Framework/interface/limited/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/limited/producerAbilityToImplementor.h @@ -34,97 +34,97 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::limited::impl::StreamCacheHolder Type; + using Type = edm::limited::impl::StreamCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::limited::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::limited::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunCacheHolder Type; + using Type = edm::limited::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::RunSummaryCacheHolder Type; + using Type = edm::limited::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::limited::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::limited::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::WatchProcessBlock Type; + using Type = edm::limited::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginProcessBlockProducer Type; + using Type = edm::limited::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndProcessBlockProducer Type; + using Type = edm::limited::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginRunProducer Type; + using Type = edm::limited::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndRunProducer Type; + using Type = edm::limited::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::BeginLuminosityBlockProducer Type; + using Type = edm::limited::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::EndLuminosityBlockProducer Type; + using Type = edm::limited::impl::EndLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::limited::impl::Accumulator Type; + using Type = edm::limited::impl::Accumulator; }; template struct SpecializeAbilityToImplementor { - typedef typename AbilityToImplementor::Type Type; + using Type = typename AbilityToImplementor::Type; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::limited::impl::EndRunSummaryProducer Type; + using Type = typename edm::limited::impl::EndRunSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::limited::impl::EmptyType Type; + using Type = edm::limited::impl::EmptyType; }; template struct SpecializeAbilityToImplementor> { - typedef typename edm::limited::impl::EndLuminosityBlockSummaryProducer Type; + using Type = typename edm::limited::impl::EndLuminosityBlockSummaryProducer; }; template struct SpecializeAbilityToImplementor { - typedef typename edm::limited::impl::EmptyType Type; + using Type = edm::limited::impl::EmptyType; }; } // namespace producer } // namespace limited diff --git a/FWCore/Framework/interface/moduleAbilities.h b/FWCore/Framework/interface/moduleAbilities.h index 332936284ecc4..ddbaf7a6c9f87 100644 --- a/FWCore/Framework/interface/moduleAbilities.h +++ b/FWCore/Framework/interface/moduleAbilities.h @@ -44,10 +44,9 @@ namespace edm { typedef T Type; }; - template + template struct InputProcessBlockCache { static constexpr module::Abilities kAbilities = module::Abilities::kInputProcessBlockCache; - typedef T Type; }; template diff --git a/FWCore/Framework/interface/one/EDAnalyzerBase.h b/FWCore/Framework/interface/one/EDAnalyzerBase.h index e21571493a3d2..88883a1129f97 100644 --- a/FWCore/Framework/interface/one/EDAnalyzerBase.h +++ b/FWCore/Framework/interface/one/EDAnalyzerBase.h @@ -93,9 +93,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implementations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDAnalyzerBase const* module, ProductRegistry* reg); @@ -115,6 +115,8 @@ namespace edm { virtual void doBeginLuminosityBlock_(LuminosityBlock const& lbp, EventSetup const& c); virtual void doEndLuminosityBlock_(LuminosityBlock const& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); + bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/one/EDFilterBase.h b/FWCore/Framework/interface/one/EDFilterBase.h index f928380e46e52..be07237025116 100644 --- a/FWCore/Framework/interface/one/EDFilterBase.h +++ b/FWCore/Framework/interface/one/EDFilterBase.h @@ -92,9 +92,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDFilterBase* module, ProductRegistry* reg) { @@ -125,6 +125,8 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); + bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/one/EDProducerBase.h b/FWCore/Framework/interface/one/EDProducerBase.h index a70d96a9ec77a..f906832e2168b 100644 --- a/FWCore/Framework/interface/one/EDProducerBase.h +++ b/FWCore/Framework/interface/one/EDProducerBase.h @@ -92,9 +92,9 @@ namespace edm { void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*); - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + void doRespondToCloseOutputFile() { clearInputProcessBlockCaches(); } void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} void registerProductsAndCallbacks(EDProducerBase* module, ProductRegistry* reg) { @@ -125,6 +125,7 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/one/OutputModuleBase.h b/FWCore/Framework/interface/one/OutputModuleBase.h index 9beb0bcc206ce..8b5a5e674e46a 100644 --- a/FWCore/Framework/interface/one/OutputModuleBase.h +++ b/FWCore/Framework/interface/one/OutputModuleBase.h @@ -33,6 +33,8 @@ #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" #include "FWCore/Framework/interface/TriggerResultsBasedEventSelector.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/ProductSelectorRules.h" @@ -90,7 +92,7 @@ namespace edm { bool selected(BranchDescription const& desc) const; - void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&); + void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&, ProcessBlockHelperBase const&); std::string const& processName() const { return process_name_; } SelectedProductsForBranchType const& keptProducts() const { return keptProducts_; } std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } @@ -118,6 +120,8 @@ namespace edm { BranchIDLists const* branchIDLists(); + OutputProcessBlockHelper const& outputProcessBlockHelper() const { return outputProcessBlockHelper_; } + ThinnedAssociationsHelper const* thinnedAssociationsHelper() const; SubProcessParentageHelper const* subProcessParentageHelper() const { return subProcessParentageHelper_; } @@ -198,6 +202,8 @@ namespace edm { edm::propagate_const> thinnedAssociationsHelper_; std::map keepAssociation_; + OutputProcessBlockHelper outputProcessBlockHelper_; + SharedResourcesAcquirer resourcesAcquirer_; SerialTaskQueue runQueue_; SerialTaskQueue luminosityBlockQueue_; @@ -208,12 +214,13 @@ namespace edm { virtual SharedResourcesAcquirer createAcquirer(); - void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) {} + void doWriteProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*); void doWriteRun(RunPrincipal const& rp, ModuleCallingContext const*, MergeableRunProductMetadata const*); void doWriteLuminosityBlock(LuminosityBlockPrincipal const& lbp, ModuleCallingContext const*); void doOpenFile(FileBlock const& fb); void doRespondToOpenInputFile(FileBlock const& fb); void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToCloseOutputFile() {} void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} std::string workerType() const { return "WorkerT"; } @@ -243,6 +250,7 @@ namespace edm { virtual void endJob() {} virtual void writeLuminosityBlock(LuminosityBlockForOutput const&) = 0; virtual void writeRun(RunForOutput const&) = 0; + virtual void writeProcessBlock(ProcessBlockForOutput const&) {} virtual void openFile(FileBlock const&) {} virtual bool isFileOpen() const { return true; } diff --git a/FWCore/Framework/interface/one/analyzerAbilityToImplementor.h b/FWCore/Framework/interface/one/analyzerAbilityToImplementor.h index 9616227d747fc..f9497511f9fd5 100644 --- a/FWCore/Framework/interface/one/analyzerAbilityToImplementor.h +++ b/FWCore/Framework/interface/one/analyzerAbilityToImplementor.h @@ -36,37 +36,37 @@ namespace edm { template <> struct AbilityToImplementor { - typedef edm::one::impl::SharedResourcesUser Type; + using Type = edm::one::impl::SharedResourcesUser; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::RunWatcher Type; + using Type = edm::one::impl::RunWatcher; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::LuminosityBlockWatcher Type; + using Type = edm::one::impl::LuminosityBlockWatcher; }; - template - struct AbilityToImplementor> { - typedef edm::one::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::one::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::RunCacheHolder Type; + using Type = edm::one::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::LuminosityBlockCacheHolder Type; + using Type = edm::one::impl::LuminosityBlockCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::WatchProcessBlock Type; + using Type = edm::one::impl::WatchProcessBlock; }; } // namespace analyzer } // namespace one diff --git a/FWCore/Framework/interface/one/filterAbilityToImplementor.h b/FWCore/Framework/interface/one/filterAbilityToImplementor.h index fe441596a8b33..e112879884d41 100644 --- a/FWCore/Framework/interface/one/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/one/filterAbilityToImplementor.h @@ -36,67 +36,67 @@ namespace edm { template <> struct AbilityToImplementor { - typedef edm::one::impl::SharedResourcesUser Type; + using Type = edm::one::impl::SharedResourcesUser; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::RunWatcher Type; + using Type = edm::one::impl::RunWatcher; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::LuminosityBlockWatcher Type; + using Type = edm::one::impl::LuminosityBlockWatcher; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::WatchProcessBlock Type; + using Type = edm::one::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginProcessBlockProducer Type; + using Type = edm::one::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndProcessBlockProducer Type; + using Type = edm::one::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginRunProducer Type; + using Type = edm::one::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndRunProducer Type; + using Type = edm::one::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginLuminosityBlockProducer Type; + using Type = edm::one::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndLuminosityBlockProducer Type; + using Type = edm::one::impl::EndLuminosityBlockProducer; }; - template - struct AbilityToImplementor> { - typedef edm::one::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::one::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::RunCacheHolder Type; + using Type = edm::one::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::LuminosityBlockCacheHolder Type; + using Type = edm::one::impl::LuminosityBlockCacheHolder; }; } // namespace filter diff --git a/FWCore/Framework/interface/one/implementors.h b/FWCore/Framework/interface/one/implementors.h index 87c6160fd7f88..7ffbe162fc8fb 100644 --- a/FWCore/Framework/interface/one/implementors.h +++ b/FWCore/Framework/interface/one/implementors.h @@ -19,13 +19,21 @@ // // system include files -#include +#include +#include #include +#include +#include +#include // user include files +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Concurrency/interface/SerialTaskQueue.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" -#include "FWCore/Concurrency/interface/SerialTaskQueue.h" +#include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" @@ -203,7 +211,7 @@ namespace edm { virtual void endLuminosityBlockProduce(edm::LuminosityBlock&, edm::EventSetup const&) = 0; }; - template + template class InputProcessBlockCacheHolder : public virtual T { public: InputProcessBlockCacheHolder() = default; @@ -211,21 +219,43 @@ namespace edm { InputProcessBlockCacheHolder& operator=(InputProcessBlockCacheHolder const&) = delete; ~InputProcessBlockCacheHolder() override {} - protected: - // Not implemented yet - // const C* inputProcessBlockCache(ProcessBlockIndex index) const { return caches_.at(index).get(); } + std::tuple...> processBlockCaches(Event const& event) const { + return cacheImpl_.processBlockCaches(event); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& func) { + cacheImpl_.template registerProcessBlockCacheFiller(token, + std::forward(func)); + } + + // This is intended for use by Framework unit tests only + unsigned int cacheSize() const { return cacheImpl_.cacheSize(); } private: - // Not yet fully implemented, will never get called - // THINK ABOUT HOW CACHES ARE CLEARED!!! + void doSelectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) final { + cacheImpl_.selectInputProcessBlocks(productRegistry, processBlockHelperBase, *this); + } + void doAccessInputProcessBlock_(ProcessBlock const& pb) final { - caches_.push_back(accessInputProcessBlock(pb)); + cacheImpl_.accessInputProcessBlock(pb); + accessInputProcessBlock(pb); } - // Not yet fully implemented, will never get called - virtual std::shared_ptr accessInputProcessBlock(ProcessBlock const&) = 0; + // Alternate method to access ProcessBlocks without using the caches + // Mostly intended for unit testing, but might have other uses... + virtual void accessInputProcessBlock(ProcessBlock const&) {} + + void clearInputProcessBlockCaches() final { cacheImpl_.clearCaches(); } - std::vector> caches_; + edm::impl::InputProcessBlockCacheImpl cacheImpl_; }; template diff --git a/FWCore/Framework/interface/one/producerAbilityToImplementor.h b/FWCore/Framework/interface/one/producerAbilityToImplementor.h index 8af9bc969db5c..85b0a64f21859 100644 --- a/FWCore/Framework/interface/one/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/one/producerAbilityToImplementor.h @@ -36,72 +36,72 @@ namespace edm { template <> struct AbilityToImplementor { - typedef edm::one::impl::SharedResourcesUser Type; + using Type = edm::one::impl::SharedResourcesUser; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::RunWatcher Type; + using Type = edm::one::impl::RunWatcher; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::LuminosityBlockWatcher Type; + using Type = edm::one::impl::LuminosityBlockWatcher; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::WatchProcessBlock Type; + using Type = edm::one::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginProcessBlockProducer Type; + using Type = edm::one::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndProcessBlockProducer Type; + using Type = edm::one::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginRunProducer Type; + using Type = edm::one::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndRunProducer Type; + using Type = edm::one::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::BeginLuminosityBlockProducer Type; + using Type = edm::one::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::EndLuminosityBlockProducer Type; + using Type = edm::one::impl::EndLuminosityBlockProducer; }; - template - struct AbilityToImplementor> { - typedef edm::one::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::one::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::RunCacheHolder Type; + using Type = edm::one::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::one::impl::LuminosityBlockCacheHolder Type; + using Type = edm::one::impl::LuminosityBlockCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::one::impl::Accumulator Type; + using Type = edm::one::impl::Accumulator; }; } // namespace producer diff --git a/FWCore/Framework/interface/processBlockUtilities.h b/FWCore/Framework/interface/processBlockUtilities.h new file mode 100644 index 0000000000000..f8b00491ff752 --- /dev/null +++ b/FWCore/Framework/interface/processBlockUtilities.h @@ -0,0 +1,19 @@ +#ifndef FWCore_Framework_processBlockUtilities_h +#define FWCore_Framework_processBlockUtilities_h + +/** + +\author W. David Dagenhart, created 13 January, 2021 + +*/ + +#include + +namespace edm { + + class Event; + + unsigned int eventProcessBlockIndex(Event const& event, std::string const& processName); + +} // namespace edm +#endif diff --git a/FWCore/Framework/interface/stream/AbilityChecker.h b/FWCore/Framework/interface/stream/AbilityChecker.h index 2c139818b29a3..c73010589f7ab 100644 --- a/FWCore/Framework/interface/stream/AbilityChecker.h +++ b/FWCore/Framework/interface/stream/AbilityChecker.h @@ -37,8 +37,8 @@ namespace edm { static constexpr bool kGlobalCache = true; }; - template - struct HasAbility, U...> : public HasAbility { + template + struct HasAbility, U...> : public HasAbility { static constexpr bool kInputProcessBlockCache = true; }; diff --git a/FWCore/Framework/interface/stream/AbilityToImplementor.h b/FWCore/Framework/interface/stream/AbilityToImplementor.h index d1d9825031615..8fc9117dd8fba 100644 --- a/FWCore/Framework/interface/stream/AbilityToImplementor.h +++ b/FWCore/Framework/interface/stream/AbilityToImplementor.h @@ -32,79 +32,79 @@ namespace edm { template struct AbilityToImplementor> { - typedef edm::stream::impl::GlobalCacheHolder Type; + using Type = edm::stream::impl::GlobalCacheHolder; }; - template - struct AbilityToImplementor> { - typedef edm::stream::impl::InputProcessBlockCacheHolder Type; + template + struct AbilityToImplementor> { + using Type = edm::stream::impl::InputProcessBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::stream::impl::RunCacheHolder Type; + using Type = edm::stream::impl::RunCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::stream::impl::RunSummaryCacheHolder Type; + using Type = edm::stream::impl::RunSummaryCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::stream::impl::LuminosityBlockCacheHolder Type; + using Type = edm::stream::impl::LuminosityBlockCacheHolder; }; template struct AbilityToImplementor> { - typedef edm::stream::impl::LuminosityBlockSummaryCacheHolder Type; + using Type = edm::stream::impl::LuminosityBlockSummaryCacheHolder; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::WatchProcessBlock Type; + using Type = edm::stream::impl::WatchProcessBlock; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::BeginProcessBlockProducer Type; + using Type = edm::stream::impl::BeginProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::EndProcessBlockProducer Type; + using Type = edm::stream::impl::EndProcessBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::BeginRunProducer Type; + using Type = edm::stream::impl::BeginRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::EndRunProducer Type; + using Type = edm::stream::impl::EndRunProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::BeginLuminosityBlockProducer Type; + using Type = edm::stream::impl::BeginLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::EndLuminosityBlockProducer Type; + using Type = edm::stream::impl::EndLuminosityBlockProducer; }; template <> struct AbilityToImplementor { - typedef edm::stream::impl::ExternalWork Type; + using Type = edm::stream::impl::ExternalWork; }; // As currently implemented this ability only works // with EDProducer, not with EDAnalyzers or EDFilters! template <> struct AbilityToImplementor { - typedef edm::stream::impl::Accumulator Type; + using Type = edm::stream::impl::Accumulator; }; } // namespace stream } // namespace edm diff --git a/FWCore/Framework/interface/stream/CacheContexts.h b/FWCore/Framework/interface/stream/CacheContexts.h index 5a13022a50498..e823fb26a0dbd 100644 --- a/FWCore/Framework/interface/stream/CacheContexts.h +++ b/FWCore/Framework/interface/stream/CacheContexts.h @@ -5,7 +5,7 @@ // Package: FWCore/Framework // Class : CacheContexts // -/**\class CacheContexts CacheContexts.h "FWCore/Framework/interface/stream/CacheContexts.h" +/**\class edm::stream::CacheContexts CacheContexts.h "FWCore/Framework/interface/stream/CacheContexts.h" Description: Helper class used to identify the caches requested by a module @@ -21,6 +21,7 @@ // system include files // user include files +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/moduleAbilities.h" // forward declarations @@ -34,36 +35,42 @@ namespace edm { template struct AbilityToCache, U...> : public AbilityToCache { - typedef G GlobalCache; + using GlobalCache = G; + }; + + template + struct AbilityToCache, U...> : public AbilityToCache { + using InputProcessBlockCache = edm::impl::InputProcessBlockCacheImpl; }; template struct AbilityToCache, U...> : public AbilityToCache { - typedef R RunCache; + using RunCache = R; }; template struct AbilityToCache, U...> : public AbilityToCache { - typedef L LuminosityBlockCache; + using LuminosityBlockCache = L; }; template struct AbilityToCache, U...> : public AbilityToCache { - typedef R RunSummaryCache; + using RunSummaryCache = R; }; template struct AbilityToCache, U...> : public AbilityToCache { - typedef L LuminosityBlockSummaryCache; + using LuminosityBlockSummaryCache = L; }; template <> struct AbilityToCache { - typedef void GlobalCache; - typedef void RunCache; - typedef void LuminosityBlockCache; - typedef void RunSummaryCache; - typedef void LuminosityBlockSummaryCache; + using GlobalCache = void; + using InputProcessBlockCache = void; + using RunCache = void; + using LuminosityBlockCache = void; + using RunSummaryCache = void; + using LuminosityBlockSummaryCache = void; }; } // namespace impl diff --git a/FWCore/Framework/interface/stream/EDAnalyzer.h b/FWCore/Framework/interface/stream/EDAnalyzer.h index f32203ca20bf8..97525ab095662 100644 --- a/FWCore/Framework/interface/stream/EDAnalyzer.h +++ b/FWCore/Framework/interface/stream/EDAnalyzer.h @@ -18,48 +18,36 @@ // Created: Thu, 01 Aug 2013 21:41:42 GMT // -// system include files - -// user include files #include "FWCore/Framework/interface/stream/AbilityToImplementor.h" #include "FWCore/Framework/interface/stream/CacheContexts.h" #include "FWCore/Framework/interface/stream/Contexts.h" #include "FWCore/Framework/interface/stream/AbilityChecker.h" #include "FWCore/Framework/interface/stream/EDAnalyzerBase.h" -// forward declarations + namespace edm { namespace stream { + template class EDAnalyzer : public AbilityToImplementor::Type..., public EDAnalyzerBase { public: - typedef CacheContexts CacheTypes; + using CacheTypes = CacheContexts; - typedef typename CacheTypes::GlobalCache GlobalCache; - typedef typename CacheTypes::RunCache RunCache; - typedef typename CacheTypes::LuminosityBlockCache LuminosityBlockCache; - typedef RunContextT RunContext; - typedef LuminosityBlockContextT LuminosityBlockContext; - typedef typename CacheTypes::RunSummaryCache RunSummaryCache; - typedef typename CacheTypes::LuminosityBlockSummaryCache LuminosityBlockSummaryCache; - - typedef AbilityChecker HasAbility; - - EDAnalyzer() = default; + using GlobalCache = typename CacheTypes::GlobalCache; + using InputProcessBlockCache = typename CacheTypes::InputProcessBlockCache; + using RunCache = typename CacheTypes::RunCache; + using LuminosityBlockCache = typename CacheTypes::LuminosityBlockCache; + using RunContext = RunContextT; + using LuminosityBlockContext = LuminosityBlockContextT; + using RunSummaryCache = typename CacheTypes::RunSummaryCache; + using LuminosityBlockSummaryCache = typename CacheTypes::LuminosityBlockSummaryCache; - // ---------- const member functions --------------------- - - // ---------- static member functions -------------------- - - // ---------- member functions --------------------------- + using HasAbility = AbilityChecker; using EDAnalyzerBase::callWhenNewProductsRegistered; - private: - EDAnalyzer(const EDAnalyzer&) = delete; // stop default - - const EDAnalyzer& operator=(const EDAnalyzer&) = delete; // stop default - - // ---------- member data -------------------------------- + EDAnalyzer() = default; + EDAnalyzer(const EDAnalyzer&) = delete; + const EDAnalyzer& operator=(const EDAnalyzer&) = delete; }; } // namespace stream diff --git a/FWCore/Framework/interface/stream/EDAnalyzerAdaptor.h b/FWCore/Framework/interface/stream/EDAnalyzerAdaptor.h index 9a50c7bafa379..6d420137af8cc 100644 --- a/FWCore/Framework/interface/stream/EDAnalyzerAdaptor.h +++ b/FWCore/Framework/interface/stream/EDAnalyzerAdaptor.h @@ -60,6 +60,8 @@ namespace edm { m_lumiSummaries.resize(1); typename T::GlobalCache const* dummy = nullptr; m_global = impl::makeGlobal(iPSet, dummy); + typename T::InputProcessBlockCache const* dummyInputProcessBlockCacheImpl = nullptr; + m_inputProcessBlocks = impl::makeInputProcessBlockCacheImpl(dummyInputProcessBlockCacheImpl); } EDAnalyzerAdaptor(const EDAnalyzerAdaptor&) = delete; // stop default const EDAnalyzerAdaptor& operator=(const EDAnalyzerAdaptor&) = delete; // stop default @@ -85,9 +87,10 @@ namespace edm { using MyGlobalLuminosityBlockSummary = CallGlobalLuminosityBlockSummary; void setupStreamModules() final { - this->createStreamModules([this]() -> EDAnalyzerBase* { + this->createStreamModules([this](unsigned int iStreamModule) -> EDAnalyzerBase* { auto tmp = impl::makeStreamModule(*m_pset, m_global.get()); MyGlobal::set(tmp, m_global.get()); + MyInputProcessBlock::set(tmp, &m_inputProcessBlocks, iStreamModule); return tmp; }); m_pset = nullptr; @@ -132,7 +135,7 @@ namespace edm { ProcessBlock processBlock(pbp, moduleDescription(), mcc, false); processBlock.setConsumer(consumer()); ProcessBlock const& cnstProcessBlock = processBlock; - MyInputProcessBlock::accessInputProcessBlock(cnstProcessBlock, m_global.get()); + MyInputProcessBlock::accessInputProcessBlock(cnstProcessBlock, m_global.get(), m_inputProcessBlocks); } } @@ -222,8 +225,17 @@ namespace edm { } } + void doRespondToCloseOutputFile() final { MyInputProcessBlock::clearCaches(m_inputProcessBlocks); } + + void selectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) final { + MyInputProcessBlock::selectInputProcessBlocks( + m_inputProcessBlocks, productRegistry, processBlockHelperBase, *consumer()); + } + // ---------- member data -------------------------------- typename impl::choose_unique_ptr::type m_global; + typename impl::choose_unique_ptr::type m_inputProcessBlocks; typename impl::choose_shared_vec::type m_runs; typename impl::choose_shared_vec::type m_lumis; typename impl::choose_shared_vec::type m_runSummaries; diff --git a/FWCore/Framework/interface/stream/EDAnalyzerAdaptorBase.h b/FWCore/Framework/interface/stream/EDAnalyzerAdaptorBase.h index 6241b287010da..95ec3a1a268f3 100644 --- a/FWCore/Framework/interface/stream/EDAnalyzerAdaptorBase.h +++ b/FWCore/Framework/interface/stream/EDAnalyzerAdaptorBase.h @@ -26,6 +26,7 @@ // user include files #include "DataFormats/Provenance/interface/BranchType.h" #include "FWCore/Utilities/interface/ProductResolverIndex.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "FWCore/ParameterSet/interface/ParameterSetfwd.h" @@ -93,9 +94,11 @@ namespace edm { protected: template void createStreamModules(T iFunc) { + unsigned int iStreamModule = 0; for (auto& m : m_streamModules) { - m = iFunc(); + m = iFunc(iStreamModule); setModuleDescriptionPtr(m); + ++iStreamModule; } } @@ -109,6 +112,7 @@ namespace edm { void updateLookup(BranchType iBranchType, ProductResolverIndexHelper const&, bool iPrefetchMayGet); void updateLookup(eventsetup::ESRecordsToProxyIndices const&); + virtual void selectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&) = 0; const EDConsumerBase* consumer() const; @@ -158,9 +162,9 @@ namespace edm { virtual void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*) = 0; virtual void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*) = 0; - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + virtual void doRespondToCloseOutputFile() = 0; void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/stream/EDFilter.h b/FWCore/Framework/interface/stream/EDFilter.h index ab3237414f0bf..2b6be0cce9b3f 100644 --- a/FWCore/Framework/interface/stream/EDFilter.h +++ b/FWCore/Framework/interface/stream/EDFilter.h @@ -18,9 +18,6 @@ // Created: Thu, 01 Aug 2013 21:41:42 GMT // -// system include files - -// user include files #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/stream/AbilityToImplementor.h" #include "FWCore/Framework/interface/stream/CacheContexts.h" @@ -28,35 +25,32 @@ #include "FWCore/Framework/interface/stream/AbilityChecker.h" #include "FWCore/Framework/interface/stream/EDFilterBase.h" #include "FWCore/Framework/interface/stream/ProducingModuleHelper.h" -// forward declarations + namespace edm { class WaitingTaskWithArenaHolder; namespace stream { + template class EDFilter : public AbilityToImplementor::Type..., public EDFilterBase { public: - typedef CacheContexts CacheTypes; + using CacheTypes = CacheContexts; - typedef typename CacheTypes::GlobalCache GlobalCache; - typedef typename CacheTypes::RunCache RunCache; - typedef typename CacheTypes::LuminosityBlockCache LuminosityBlockCache; - typedef RunContextT RunContext; - typedef LuminosityBlockContextT LuminosityBlockContext; - typedef typename CacheTypes::RunSummaryCache RunSummaryCache; - typedef typename CacheTypes::LuminosityBlockSummaryCache LuminosityBlockSummaryCache; + using GlobalCache = typename CacheTypes::GlobalCache; + using InputProcessBlockCache = typename CacheTypes::InputProcessBlockCache; + using RunCache = typename CacheTypes::RunCache; + using LuminosityBlockCache = typename CacheTypes::LuminosityBlockCache; + using RunContext = RunContextT; + using LuminosityBlockContext = LuminosityBlockContextT; + using RunSummaryCache = typename CacheTypes::RunSummaryCache; + using LuminosityBlockSummaryCache = typename CacheTypes::LuminosityBlockSummaryCache; - typedef AbilityChecker HasAbility; + using HasAbility = AbilityChecker; EDFilter() = default; - //virtual ~EDFilter(); - - // ---------- const member functions --------------------- - - // ---------- static member functions -------------------- - - // ---------- member functions --------------------------- + EDFilter(const EDFilter&) = delete; + const EDFilter& operator=(const EDFilter&) = delete; bool hasAbilityToProduceInBeginProcessBlocks() const final { return HasAbilityToProduceInBeginProcessBlocks::value; @@ -72,15 +66,9 @@ namespace edm { bool hasAbilityToProduceInEndLumis() const final { return HasAbilityToProduceInEndLumis::value; } private: - EDFilter(const EDFilter&) = delete; // stop default - - const EDFilter& operator=(const EDFilter&) = delete; // stop default - void doAcquire_(Event const& ev, EventSetup const& es, WaitingTaskWithArenaHolder& holder) final { doAcquireIfNeeded(this, ev, es, holder); } - - // ---------- member data -------------------------------- }; } // namespace stream diff --git a/FWCore/Framework/interface/stream/EDProducer.h b/FWCore/Framework/interface/stream/EDProducer.h index 69f751735fdcb..eae7abd0cb267 100644 --- a/FWCore/Framework/interface/stream/EDProducer.h +++ b/FWCore/Framework/interface/stream/EDProducer.h @@ -18,9 +18,6 @@ // Created: Thu, 01 Aug 2013 21:41:42 GMT // -// system include files - -// user include files #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/stream/AbilityToImplementor.h" #include "FWCore/Framework/interface/stream/CacheContexts.h" @@ -28,29 +25,35 @@ #include "FWCore/Framework/interface/stream/AbilityChecker.h" #include "FWCore/Framework/interface/stream/EDProducerBase.h" #include "FWCore/Framework/interface/stream/ProducingModuleHelper.h" -// forward declarations + namespace edm { class WaitingTaskWithArenaHolder; namespace stream { + template class EDProducer : public AbilityToImplementor::Type..., public std::conditional::kHasIt, impl::EmptyType, EDProducerBase>::type { public: - typedef CacheContexts CacheTypes; + using CacheTypes = CacheContexts; - typedef typename CacheTypes::GlobalCache GlobalCache; - typedef typename CacheTypes::RunCache RunCache; - typedef typename CacheTypes::LuminosityBlockCache LuminosityBlockCache; - typedef RunContextT RunContext; - typedef LuminosityBlockContextT LuminosityBlockContext; - typedef typename CacheTypes::RunSummaryCache RunSummaryCache; - typedef typename CacheTypes::LuminosityBlockSummaryCache LuminosityBlockSummaryCache; + using GlobalCache = typename CacheTypes::GlobalCache; + using InputProcessBlockCache = typename CacheTypes::InputProcessBlockCache; + using RunCache = typename CacheTypes::RunCache; + using LuminosityBlockCache = typename CacheTypes::LuminosityBlockCache; + using RunContext = RunContextT; + using LuminosityBlockContext = LuminosityBlockContextT; + using RunSummaryCache = typename CacheTypes::RunSummaryCache; + using LuminosityBlockSummaryCache = typename CacheTypes::LuminosityBlockSummaryCache; - typedef AbilityChecker HasAbility; + using HasAbility = AbilityChecker; + + EDProducer() = default; + EDProducer(const EDProducer&) = delete; + const EDProducer& operator=(const EDProducer&) = delete; bool hasAbilityToProduceInBeginProcessBlocks() const final { return HasAbilityToProduceInBeginProcessBlocks::value; @@ -65,25 +68,10 @@ namespace edm { bool hasAbilityToProduceInBeginLumis() const final { return HasAbilityToProduceInBeginLumis::value; } bool hasAbilityToProduceInEndLumis() const final { return HasAbilityToProduceInEndLumis::value; } - EDProducer() = default; - //virtual ~EDProducer(); - - // ---------- const member functions --------------------- - - // ---------- static member functions -------------------- - - // ---------- member functions --------------------------- - private: - EDProducer(const EDProducer&) = delete; // stop default - - const EDProducer& operator=(const EDProducer&) = delete; // stop default - void doAcquire_(Event const& ev, EventSetup const& es, WaitingTaskWithArenaHolder& holder) final { doAcquireIfNeeded(this, ev, es, holder); } - - // ---------- member data -------------------------------- }; } // namespace stream diff --git a/FWCore/Framework/interface/stream/ProducingModuleAdaptor.h b/FWCore/Framework/interface/stream/ProducingModuleAdaptor.h index b2ac3bc3c20be..374eca94e63f6 100644 --- a/FWCore/Framework/interface/stream/ProducingModuleAdaptor.h +++ b/FWCore/Framework/interface/stream/ProducingModuleAdaptor.h @@ -48,6 +48,8 @@ namespace edm { m_lumiSummaries.resize(1); typename T::GlobalCache const* dummy = nullptr; m_global = impl::makeGlobal(iPSet, dummy); + typename T::InputProcessBlockCache const* dummyInputProcessBlockCacheImpl = nullptr; + m_inputProcessBlocks = impl::makeInputProcessBlockCacheImpl(dummyInputProcessBlockCacheImpl); } ProducingModuleAdaptor(const ProducingModuleAdaptor&) = delete; // stop default const ProducingModuleAdaptor& operator=(const ProducingModuleAdaptor&) = delete; // stop default @@ -90,9 +92,10 @@ namespace edm { using MyEndLuminosityBlockProduce = CallEndLuminosityBlockProduce; void setupStreamModules() final { - this->createStreamModules([this]() -> M* { + this->createStreamModules([this](unsigned int iStreamModule) -> M* { auto tmp = impl::makeStreamModule(*m_pset, m_global.get()); MyGlobal::set(tmp, m_global.get()); + MyInputProcessBlock::set(tmp, &m_inputProcessBlocks, iStreamModule); return tmp; }); m_pset = nullptr; @@ -141,7 +144,7 @@ namespace edm { ProcessBlock processBlock(pbp, this->moduleDescription(), mcc, false); ProcessBlock const& cnstProcessBlock = processBlock; processBlock.setConsumer(this->consumer()); - MyInputProcessBlock::accessInputProcessBlock(cnstProcessBlock, m_global.get()); + MyInputProcessBlock::accessInputProcessBlock(cnstProcessBlock, m_global.get(), m_inputProcessBlocks); } } @@ -260,8 +263,19 @@ namespace edm { } } + void doRespondToCloseOutputFile() final { MyInputProcessBlock::clearCaches(m_inputProcessBlocks); } + + using B::consumer; + + void selectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) final { + MyInputProcessBlock::selectInputProcessBlocks( + m_inputProcessBlocks, productRegistry, processBlockHelperBase, *consumer()); + } + // ---------- member data -------------------------------- typename impl::choose_unique_ptr::type m_global; + typename impl::choose_unique_ptr::type m_inputProcessBlocks; typename impl::choose_shared_vec::type m_runs; typename impl::choose_shared_vec::type m_lumis; typename impl::choose_shared_vec::type m_runSummaries; diff --git a/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h b/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h index 3d304acb939b3..2171e04ab8ba4 100644 --- a/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h +++ b/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h @@ -27,6 +27,7 @@ // user include files #include "DataFormats/Provenance/interface/BranchType.h" #include "FWCore/Utilities/interface/ProductResolverIndex.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "FWCore/ParameterSet/interface/ParameterSetfwd.h" @@ -102,6 +103,7 @@ namespace edm { void updateLookup(BranchType iBranchType, ProductResolverIndexHelper const&, bool iPrefetchMayGet); void updateLookup(eventsetup::ESRecordsToProxyIndices const&); + virtual void selectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&) = 0; void modulesWhoseProductsAreConsumed(std::array*, NumBranchTypes>& modules, std::vector& modulesInPreviousProcesses, @@ -125,9 +127,11 @@ namespace edm { protected: template void createStreamModules(F iFunc) { + unsigned int iStreamModule = 0; for (auto& m : m_streamModules) { - m = iFunc(); + m = iFunc(iStreamModule); m->setModuleDescriptionPtr(&moduleDescription_); + ++iStreamModule; } } @@ -172,9 +176,9 @@ namespace edm { virtual void doBeginLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*) = 0; virtual void doEndLuminosityBlock(LumiTransitionInfo const&, ModuleCallingContext const*) = 0; - //For now, the following are just dummy implemenations with no ability for users to override - void doRespondToOpenInputFile(FileBlock const& fb); - void doRespondToCloseInputFile(FileBlock const& fb); + void doRespondToOpenInputFile(FileBlock const&) {} + void doRespondToCloseInputFile(FileBlock const&) {} + virtual void doRespondToCloseOutputFile() = 0; void doRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&); // ---------- member data -------------------------------- diff --git a/FWCore/Framework/interface/stream/callAbilities.h b/FWCore/Framework/interface/stream/callAbilities.h index 966e2e4bba12c..df20d8c94d71f 100644 --- a/FWCore/Framework/interface/stream/callAbilities.h +++ b/FWCore/Framework/interface/stream/callAbilities.h @@ -21,11 +21,15 @@ // system include files #include #include + // user include files +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/stream/dummy_helpers.h" // forward declarations namespace edm { + class EDConsumerBase; class Run; class EventSetup; class LuminosityBlock; @@ -69,28 +73,99 @@ namespace edm { //******************************** template struct CallInputProcessBlockImpl { - static void accessInputProcessBlock(edm::ProcessBlock const& iProcessBlock, typename T::GlobalCache* iGC) { - // This is not fully implemented yet and will never be called - T::accessInputProcessBlock(iProcessBlock, iGC); + static void set(T* iProd, + typename impl::choose_unique_ptr::type const* iCaches, + unsigned int iStreamModule) { + iProd->setProcessBlockCache(iCaches->get()); + if (iStreamModule == 0 && iProd->cacheFillersRegistered()) { + (*iCaches)->copyProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); + } + iProd->clearRegistration(); + } + + static void selectInputProcessBlocks( + typename impl::choose_unique_ptr::type& iCaches, + ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase, + EDConsumerBase const& edConsumerBase) { + iCaches->selectInputProcessBlocks(productRegistry, processBlockHelperBase, edConsumerBase); + } + + static void accessInputProcessBlock( + edm::ProcessBlock const& processBlock, + typename T::GlobalCache* iGC, + typename impl::choose_unique_ptr::type& iCaches) { + iCaches->accessInputProcessBlock(processBlock); + T::accessInputProcessBlock(processBlock, iGC); + } + + static void clearCaches(typename impl::choose_unique_ptr::type& iCaches) { + iCaches->clearCaches(); } }; template struct CallInputProcessBlockImpl { - static void accessInputProcessBlock(edm::ProcessBlock const& processBlock, typename T::GlobalCache*) { - // This is not fully implemented yet and will never be called + static void set(T* iProd, + typename impl::choose_unique_ptr::type const* iCaches, + unsigned int iStreamModule) { + iProd->setProcessBlockCache(iCaches->get()); + if (iStreamModule == 0 && iProd->cacheFillersRegistered()) { + (*iCaches)->copyProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); + } + iProd->clearRegistration(); + } + + static void selectInputProcessBlocks( + typename impl::choose_unique_ptr::type& iCaches, + ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase, + EDConsumerBase const& edConsumerBase) { + iCaches->selectInputProcessBlocks(productRegistry, processBlockHelperBase, edConsumerBase); + } + + static void accessInputProcessBlock( + edm::ProcessBlock const& processBlock, + typename T::GlobalCache*, + typename impl::choose_unique_ptr::type& iCaches) { + iCaches->accessInputProcessBlock(processBlock); T::accessInputProcessBlock(processBlock); } + + static void clearCaches(typename impl::choose_unique_ptr::type& iCaches) { + iCaches->clearCaches(); + } }; template struct CallInputProcessBlockImpl { - static void accessInputProcessBlock(edm::ProcessBlock const&, typename T::GlobalCache*) {} + static void set(void*, void const*, unsigned int) {} + static void selectInputProcessBlocks(typename impl::choose_unique_ptr::type&, + ProductRegistry const&, + ProcessBlockHelperBase const&, + EDConsumerBase const&) {} + + static void accessInputProcessBlock( + edm::ProcessBlock const&, + typename T::GlobalCache*, + typename impl::choose_unique_ptr::type& iCaches) {} + + static void clearCaches(typename impl::choose_unique_ptr::type&) {} }; template struct CallInputProcessBlockImpl { - static void accessInputProcessBlock(edm::ProcessBlock const&, typename T::GlobalCache*) {} + static void set(void*, void const*, unsigned int) {} + static void selectInputProcessBlocks(typename impl::choose_unique_ptr::type&, + ProductRegistry const&, + ProcessBlockHelperBase const&, + EDConsumerBase const&) {} + static void accessInputProcessBlock( + edm::ProcessBlock const&, + typename T::GlobalCache*, + typename impl::choose_unique_ptr::type& iCaches) {} + + static void clearCaches(typename impl::choose_unique_ptr::type&) {} }; template diff --git a/FWCore/Framework/interface/stream/implementors.h b/FWCore/Framework/interface/stream/implementors.h index be2482d9e1470..021e561e2671f 100644 --- a/FWCore/Framework/interface/stream/implementors.h +++ b/FWCore/Framework/interface/stream/implementors.h @@ -19,14 +19,24 @@ // // system include files +#include +#include #include +#include +#include +#include // user include files +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/stream/EDProducerBase.h" #include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/RunIndex.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" +#include "FWCore/Utilities/interface/TypeID.h" // forward declarations namespace edm { @@ -53,19 +63,96 @@ namespace edm { C const* cache_; }; - template + template class InputProcessBlockCacheHolder { public: InputProcessBlockCacheHolder() = default; InputProcessBlockCacheHolder(InputProcessBlockCacheHolder const&) = delete; InputProcessBlockCacheHolder& operator=(InputProcessBlockCacheHolder const&) = delete; - //void setProcessBlockCache(C const* iCache) { cache_ = iCache; } - protected: - //C const* inputProcessBlockCache() const { return cache_; } + std::tuple...> processBlockCaches(Event const& event) const { + return cacheImpl_->processBlockCaches(event); + } + + template + using CacheTypeT = typename std::tuple_element>::type; + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& cacheFiller) { + registerProcessBlockCacheFiller, DataType, Func>( + token, std::forward(cacheFiller)); + } + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& cacheFiller) { + static_assert(edm::impl::countTypeInParameterPack() == 1u, + "If registerProcessBlockCacheFiller is called with a type template parameter\n" + "then that type must appear exactly once in the template parameters of InputProcessBlockCache"); + + // Find the index into the parameter pack from the CacheType + constexpr unsigned int I = edm::impl::indexInputProcessBlockCache(); + + registerProcessBlockCacheFiller(token, std::forward(cacheFiller)); + } private: - //C const* cache_; + template + friend struct edm::stream::CallInputProcessBlockImpl; + + void setProcessBlockCache(edm::impl::InputProcessBlockCacheImpl const* cacheImpl) { + cacheImpl_ = cacheImpl; + } + + bool cacheFillersRegistered() const { return registrationInfo_ ? true : false; } + std::vector& tokenInfos() { return registrationInfo_->tokenInfos_; } + std::tuple...>& cacheFillers() { return registrationInfo_->cacheFillers_; } + + void clearRegistration() { registrationInfo_.reset(); } + + // The next two functions exist so that it is optional whether modules + // with this ability implement them. + + static void accessInputProcessBlock(edm::ProcessBlock const&) {} + + template + static void accessInputProcessBlock(edm::ProcessBlock const&, GlobalCacheType*) {} + + template + void registerProcessBlockCacheFiller(EDGetTokenT const& token, Func&& cacheFiller) { + if (!registrationInfo_) { + registrationInfo_ = std::make_unique(); + tokenInfos().resize(sizeof...(CacheTypes)); + } + + if (!tokenInfos()[ICacheType].token_.isUninitialized()) { + throw Exception(errors::LogicError) + << "registerProcessBlockCacheFiller should only be called once per cache type"; + } + + tokenInfos()[ICacheType] = edm::impl::TokenInfo{EDGetToken(token), TypeID(typeid(DataType))}; + + std::get(cacheFillers()).func_ = + std::function(ProcessBlock const&, std::shared_ptr const&)>( + std::forward(cacheFiller)); + } + + // ------------ Data members -------------------- + + edm::impl::InputProcessBlockCacheImpl const* cacheImpl_; + + // The RegistrationInfo is filled while the module constructor runs. + // Later this information is copied to the InputProcessBlockCacheImpl + // object owned by the adaptor and then registrationInfo_ is cleared. + // Note that this is really only needed for one of the stream instances, + // but we fill for all streams so registerProcessBlockCacheFiller can + // be called in the constructor. This keeps the interface as simple as + // possible and makes it similar to the consumes interface. + class RegistrationInfo { + public: + std::vector tokenInfos_; + std::tuple...> cacheFillers_; + }; + std::unique_ptr registrationInfo_; }; template diff --git a/FWCore/Framework/interface/stream/makeGlobal.h b/FWCore/Framework/interface/stream/makeGlobal.h index ac651102099cf..579ea45d962f5 100644 --- a/FWCore/Framework/interface/stream/makeGlobal.h +++ b/FWCore/Framework/interface/stream/makeGlobal.h @@ -18,11 +18,9 @@ // Created: Thu, 22 May 2014 13:55:01 GMT // -// system include files #include -// user include files + #include "FWCore/Framework/interface/stream/dummy_helpers.h" -// forward declarations namespace edm { class ParameterSet; @@ -46,6 +44,14 @@ namespace edm { T* makeStreamModule(edm::ParameterSet const& iPSet, void const*) { return new T(iPSet); } + + template + inline std::unique_ptr makeInputProcessBlockCacheImpl(G const*) { + return std::make_unique(); + } + + inline dummy_ptr makeInputProcessBlockCacheImpl(void const*) { return dummy_ptr(); } + } // namespace impl } // namespace stream } // namespace edm diff --git a/FWCore/Framework/src/EDConsumerBase.cc b/FWCore/Framework/src/EDConsumerBase.cc index 6e0ba68161203..63433b1222faa 100644 --- a/FWCore/Framework/src/EDConsumerBase.cc +++ b/FWCore/Framework/src/EDConsumerBase.cc @@ -416,6 +416,8 @@ void EDConsumerBase::throwESConsumesInProcessBlock() const { << "EventSetup products can only be consumed in Event, Lumi, or Run transitions.\n"; } +void EDConsumerBase::doSelectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&) {} + namespace { struct CharStarComp { bool operator()(const char* iLHS, const char* iRHS) const { return strcmp(iLHS, iRHS) < 0; } diff --git a/FWCore/Framework/src/EventForOutput.cc b/FWCore/Framework/src/EventForOutput.cc index 43900553b11ba..74a8a2d8c55e8 100644 --- a/FWCore/Framework/src/EventForOutput.cc +++ b/FWCore/Framework/src/EventForOutput.cc @@ -45,4 +45,8 @@ namespace edm { BranchListIndexes const& EventForOutput::branchListIndexes() const { return eventPrincipal().branchListIndexes(); } + EventToProcessBlockIndexes const& EventForOutput::eventToProcessBlockIndexes() const { + return eventPrincipal().eventToProcessBlockIndexes(); + } + } // namespace edm diff --git a/FWCore/Framework/src/EventPrincipal.cc b/FWCore/Framework/src/EventPrincipal.cc index ff4bd4360eb63..089aec3b05d46 100644 --- a/FWCore/Framework/src/EventPrincipal.cc +++ b/FWCore/Framework/src/EventPrincipal.cc @@ -13,6 +13,7 @@ #include "DataFormats/Provenance/interface/ProductRegistry.h" #include "DataFormats/Provenance/interface/Provenance.h" #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelperBase.h" #include "FWCore/Framework/interface/DelayedReader.h" #include "FWCore/Framework/interface/ProductResolverBase.h" #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" @@ -35,13 +36,15 @@ namespace edm { ProcessConfiguration const& pc, HistoryAppender* historyAppender, unsigned int streamIndex, - bool isForPrimaryProcess) + bool isForPrimaryProcess, + ProcessBlockHelperBase const* processBlockHelper) : Base(reg, reg->productLookup(InEvent), pc, InEvent, historyAppender, isForPrimaryProcess), aux_(), luminosityBlockPrincipal_(nullptr), provRetrieverPtr_(new ProductProvenanceRetriever(streamIndex, *reg)), eventSelectionIDs_(), branchIDListHelper_(branchIDListHelper), + processBlockHelper_(processBlockHelper), thinnedAssociationsHelper_(thinnedAssociationsHelper), branchListIndexes_(), branchListIndexToProcessIndex_(), @@ -67,6 +70,7 @@ namespace edm { ProcessHistory const* processHistory, EventSelectionIDVector eventSelectionIDs, BranchListIndexes branchListIndexes, + EventToProcessBlockIndexes const& eventToProcessBlockIndexes, ProductProvenanceRetriever const& provRetriever, DelayedReader* reader, bool deepCopyRetriever) { @@ -83,6 +87,7 @@ namespace edm { } updateBranchListIndexes(std::move(branchListIndexes)); } + eventToProcessBlockIndexes_ = eventToProcessBlockIndexes; commonFillEventPrincipal(aux, processHistory, reader); } @@ -252,6 +257,10 @@ namespace edm { return ProductID(); } + unsigned int EventPrincipal::processBlockIndex(std::string const& processName) const { + return processBlockHelper_->processBlockIndex(processName, eventToProcessBlockIndexes_); + } + unsigned int EventPrincipal::transitionIndex_() const { return streamID_.value(); } void EventPrincipal::changedIndexes_() { provRetrieverPtr_->update(productRegistry()); } @@ -366,6 +375,10 @@ namespace edm { BranchListIndexes const& EventPrincipal::branchListIndexes() const { return branchListIndexes_; } + EventToProcessBlockIndexes const& EventPrincipal::eventToProcessBlockIndexes() const { + return eventToProcessBlockIndexes_; + } + edm::ThinnedAssociation const* EventPrincipal::getThinnedAssociation(edm::BranchID const& branchID) const { ConstProductResolverPtr const phb = getProductResolver(branchID); diff --git a/FWCore/Framework/src/EventProcessor.cc b/FWCore/Framework/src/EventProcessor.cc index 5a0f90ee37f33..fe0a32fc92293 100644 --- a/FWCore/Framework/src/EventProcessor.cc +++ b/FWCore/Framework/src/EventProcessor.cc @@ -6,6 +6,7 @@ #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" #include "DataFormats/Provenance/interface/SubProcessParentageHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" #include "FWCore/Framework/src/CommonParams.h" #include "FWCore/Framework/interface/EDLooperBase.h" #include "FWCore/Framework/interface/EventPrincipal.h" @@ -130,6 +131,7 @@ namespace edm { CommonParams const& common, std::shared_ptr preg, std::shared_ptr branchIDListHelper, + std::shared_ptr const& processBlockHelper, std::shared_ptr thinnedAssociationsHelper, std::shared_ptr areg, std::shared_ptr processConfiguration, @@ -174,6 +176,7 @@ namespace edm { InputSourceDescription isdesc(md, preg, branchIDListHelper, + processBlockHelper, thinnedAssociationsHelper, areg, common.maxEventsInput_, @@ -485,18 +488,22 @@ namespace edm { streamQueues_.resize(nStreams); streamLumiStatus_.resize(nStreams); + processBlockHelper_ = std::make_shared(); + // initialize the input source input_ = makeInput(*parameterSet, *common, items.preg(), items.branchIDListHelper(), + get_underlying_safe(processBlockHelper_), items.thinnedAssociationsHelper(), items.actReg_, items.processConfiguration(), preallocations_); - // intialize the Schedule - schedule_ = items.initSchedule(*parameterSet, hasSubProcesses, preallocations_, &processContext_); + // initialize the Schedule + schedule_ = + items.initSchedule(*parameterSet, hasSubProcesses, preallocations_, &processContext_, *processBlockHelper_); // set the data members act_table_ = std::move(items.act_table_); @@ -519,7 +526,9 @@ namespace edm { thinnedAssociationsHelper(), *processConfiguration_, historyAppender_.get(), - index); + index, + true /*primary process*/, + &*processBlockHelper_); principalCache_.insert(std::move(ep)); } @@ -544,6 +553,7 @@ namespace edm { *parameterSet, preg(), branchIDListHelper(), + *processBlockHelper_, *thinnedAssociationsHelper_, SubProcessParentageHelper(), *espController_, @@ -663,7 +673,7 @@ namespace edm { throw; } espController_->finishConfiguration(); - schedule_->beginJob(*preg_, esp_->recordsToProxyIndices()); + schedule_->beginJob(*preg_, esp_->recordsToProxyIndices(), *processBlockHelper_); if (looper_) { constexpr bool mustPrefetchMayGet = true; auto const processBlockLookup = preg_->productLookup(InProcess); @@ -889,7 +899,7 @@ namespace edm { void EventProcessor::closeOutputFiles() { schedule_->closeOutputFiles(); for_all(subProcesses_, [](auto& subProcess) { subProcess.closeOutputFiles(); }); - + processBlockHelper_->clearAfterOutputFilesClose(); FDEBUG(1) << "\tcloseOutputFiles\n"; } @@ -991,15 +1001,10 @@ namespace edm { } void EventProcessor::inputProcessBlocks() { + input_->fillProcessBlockHelper(); ProcessBlockPrincipal& processBlockPrincipal = principalCache_.inputProcessBlockPrincipal(); - // For now the input source always returns false from readProcessBlock, - // so this does nothing at all. - // Eventually the ProcessBlockPrincipal needs to be properly filled - // and cleared. The delayed reader needs to be set. The correct process name - // needs to be supplied. - while (input_->readProcessBlock()) { - DelayedReader* reader = nullptr; - processBlockPrincipal.fillProcessBlockPrincipal(processConfiguration_->processName(), reader); + while (input_->nextProcessBlock(processBlockPrincipal)) { + readProcessBlock(processBlockPrincipal); using Traits = OccurrenceTraits; FinalWaitingTask globalWaitTask; @@ -1689,6 +1694,12 @@ namespace edm { } } + void EventProcessor::readProcessBlock(ProcessBlockPrincipal& processBlockPrincipal) { + SendSourceTerminationSignalIfException sentry(actReg_.get()); + input_->readProcessBlock(processBlockPrincipal); + sentry.completedSuccessfully(); + } + std::pair EventProcessor::readRun() { if (principalCache_.hasRunPrincipal()) { throw edm::Exception(edm::errors::LogicError) << "EventProcessor::readRun\n" diff --git a/FWCore/Framework/src/FileBlock.cc b/FWCore/Framework/src/FileBlock.cc new file mode 100644 index 0000000000000..8fbea961326db --- /dev/null +++ b/FWCore/Framework/src/FileBlock.cc @@ -0,0 +1,38 @@ +#include "FWCore/Framework/interface/FileBlock.h" + +#include + +namespace edm { + void FileBlock::updateTTreePointers(TTree* ev, + TTree* meta, + TTree* lumi, + TTree* lumiMeta, + TTree* run, + TTree* runMeta, + std::vector&& processBlockTrees, + std::vector&& processesWithProcessBlockTrees) { + tree_ = ev; + metaTree_ = meta; + lumiTree_ = lumi; + lumiMetaTree_ = lumiMeta; + runTree_ = run; + runMetaTree_ = runMeta; + processBlockTrees_ = std::move(processBlockTrees); + processesWithProcessBlockTrees_ = std::move(processesWithProcessBlockTrees); + } + + TTree* FileBlock::processBlockTree(std::string const& processName) const { + auto it = std::find(processesWithProcessBlockTrees_.begin(), processesWithProcessBlockTrees_.end(), processName); + if (it != processesWithProcessBlockTrees_.end()) { + auto index = std::distance(processesWithProcessBlockTrees_.begin(), it); + return processBlockTrees_[index]; + } + return nullptr; + } + + void FileBlock::close() { + runMetaTree_ = lumiMetaTree_ = metaTree_ = runTree_ = lumiTree_ = tree_ = nullptr; + std::fill(processBlockTrees_.begin(), processBlockTrees_.end(), nullptr); + } + +} // namespace edm diff --git a/FWCore/Framework/src/GlobalSchedule.cc b/FWCore/Framework/src/GlobalSchedule.cc index 177272192850a..4d831c798a9aa 100644 --- a/FWCore/Framework/src/GlobalSchedule.cc +++ b/FWCore/Framework/src/GlobalSchedule.cc @@ -91,8 +91,9 @@ namespace edm { void GlobalSchedule::endJob(ExceptionCollector& collector) { workerManagers_[0].endJob(collector); } void GlobalSchedule::beginJob(ProductRegistry const& iRegistry, - eventsetup::ESRecordsToProxyIndices const& iESIndices) { - workerManagers_[0].beginJob(iRegistry, iESIndices); + eventsetup::ESRecordsToProxyIndices const& iESIndices, + ProcessBlockHelperBase const& processBlockHelperBase) { + workerManagers_[0].beginJob(iRegistry, iESIndices, processBlockHelperBase); } void GlobalSchedule::replaceModule(maker::ModuleHolder* iMod, std::string const& iLabel) { diff --git a/FWCore/Framework/src/GlobalSchedule.h b/FWCore/Framework/src/GlobalSchedule.h index a21d9eaf86cfc..43e8b24a0f38c 100644 --- a/FWCore/Framework/src/GlobalSchedule.h +++ b/FWCore/Framework/src/GlobalSchedule.h @@ -5,6 +5,7 @@ */ #include "DataFormats/Provenance/interface/ModuleDescription.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/interface/ExceptionActions.h" #include "FWCore/Framework/interface/ExceptionHelpers.h" @@ -104,7 +105,7 @@ namespace edm { ServiceToken const& token, bool cleaningUpAfterException = false); - void beginJob(ProductRegistry const&, eventsetup::ESRecordsToProxyIndices const&); + void beginJob(ProductRegistry const&, eventsetup::ESRecordsToProxyIndices const&, ProcessBlockHelperBase const&); void endJob(ExceptionCollector& collector); /// Return a vector allowing const access to all the diff --git a/FWCore/Framework/src/InputSource.cc b/FWCore/Framework/src/InputSource.cc index a2c389bfe7d6b..50e8dbaaae9f1 100644 --- a/FWCore/Framework/src/InputSource.cc +++ b/FWCore/Framework/src/InputSource.cc @@ -10,6 +10,7 @@ #include "FWCore/Framework/interface/FileBlock.h" #include "FWCore/Framework/interface/InputSourceDescription.h" #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" +#include "FWCore/Framework/interface/ProcessBlockPrincipal.h" #include "FWCore/Framework/interface/RunPrincipal.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" @@ -56,6 +57,7 @@ namespace edm { productRegistry_(desc.productRegistry_), processHistoryRegistry_(new ProcessHistoryRegistry), branchIDListHelper_(desc.branchIDListHelper_), + processBlockHelper_(desc.processBlockHelper_), thinnedAssociationsHelper_(desc.thinnedAssociationsHelper_), processGUID_(createGlobalIdentifier(true)), time_(), @@ -217,10 +219,10 @@ namespace edm { void InputSource::registerProducts() {} // Return a dummy file block. - std::unique_ptr InputSource::readFile() { + std::shared_ptr InputSource::readFile() { assert(state_ == IsFile); assert(!limitReached()); - return callWithTryCatchAndPrint >([this]() { return readFile_(); }, + return callWithTryCatchAndPrint >([this]() { return readFile_(); }, "Calling InputSource::readFile_"); } @@ -235,7 +237,7 @@ namespace edm { // Return a dummy file block. // This function must be overridden for any input source that reads a file // containing Products. - std::unique_ptr InputSource::readFile_() { return std::make_unique(); } + std::shared_ptr InputSource::readFile_() { return std::make_shared(); } void InputSource::readRun(RunPrincipal& runPrincipal, HistoryAppender&) { RunSourceSentry sentry(*this, runPrincipal.index()); @@ -266,7 +268,23 @@ namespace edm { } } - bool InputSource::readProcessBlock() { return false; } + void InputSource::fillProcessBlockHelper() { fillProcessBlockHelper_(); } + + bool InputSource::nextProcessBlock(ProcessBlockPrincipal& processBlockPrincipal) { + return nextProcessBlock_(processBlockPrincipal); + } + + void InputSource::readProcessBlock(ProcessBlockPrincipal& processBlockPrincipal) { + ProcessBlockSourceSentry sentry(*this, processBlockPrincipal.processName()); + callWithTryCatchAndPrint([this, &processBlockPrincipal]() { readProcessBlock_(processBlockPrincipal); }, + "Calling InputSource::readProcessBlock_"); + } + + void InputSource::fillProcessBlockHelper_() {} + + bool InputSource::nextProcessBlock_(ProcessBlockPrincipal&) { return false; } + + void InputSource::readProcessBlock_(ProcessBlockPrincipal&) {} void InputSource::readRun_(RunPrincipal& runPrincipal) { // Note: For the moment, we do not support saving and restoring the state of the @@ -467,6 +485,16 @@ namespace edm { InputSource::RunSourceSentry::~RunSourceSentry() { source_.actReg()->postSourceRunSignal_(index_); } + InputSource::ProcessBlockSourceSentry::ProcessBlockSourceSentry(InputSource const& source, + std::string const& processName) + : source_(source), processName_(processName) { + source_.actReg()->preSourceProcessBlockSignal_(); + } + + InputSource::ProcessBlockSourceSentry::~ProcessBlockSourceSentry() { + source_.actReg()->postSourceProcessBlockSignal_(processName_); + } + InputSource::FileOpenSentry::FileOpenSentry(InputSource const& source, std::string const& lfn, bool usedFallback) : post_(source.actReg()->postOpenFileSignal_), lfn_(lfn), usedFallback_(usedFallback) { source.actReg()->preOpenFileSignal_(lfn, usedFallback); diff --git a/FWCore/Framework/src/OutputModuleCommunicator.h b/FWCore/Framework/src/OutputModuleCommunicator.h index 816bf97651dc8..a0fbdb6d54ec5 100644 --- a/FWCore/Framework/src/OutputModuleCommunicator.h +++ b/FWCore/Framework/src/OutputModuleCommunicator.h @@ -25,6 +25,7 @@ // user include files #include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" // forward declarations @@ -76,7 +77,9 @@ namespace edm { virtual SelectedProductsForBranchType const& keptProducts() const = 0; - virtual void selectProducts(ProductRegistry const& preg, ThinnedAssociationsHelper const&) = 0; + virtual void selectProducts(ProductRegistry const& preg, + ThinnedAssociationsHelper const&, + ProcessBlockHelperBase const&) = 0; virtual void setEventSelectionInfo( std::map > > const& outputModulePathPositions, diff --git a/FWCore/Framework/src/OutputModuleCommunicatorT.cc b/FWCore/Framework/src/OutputModuleCommunicatorT.cc index 3d99ac5452c56..8bf00d3ff3e66 100644 --- a/FWCore/Framework/src/OutputModuleCommunicatorT.cc +++ b/FWCore/Framework/src/OutputModuleCommunicatorT.cc @@ -198,8 +198,9 @@ namespace edm { template void OutputModuleCommunicatorT::selectProducts(edm::ProductRegistry const& preg, - ThinnedAssociationsHelper const& helper) { - module().selectProducts(preg, helper); + ThinnedAssociationsHelper const& helper, + ProcessBlockHelperBase const& processBlockHelper) { + module().selectProducts(preg, helper, processBlockHelper); } template diff --git a/FWCore/Framework/src/OutputModuleCommunicatorT.h b/FWCore/Framework/src/OutputModuleCommunicatorT.h index 5ed35b3042deb..b4ee6165a646e 100644 --- a/FWCore/Framework/src/OutputModuleCommunicatorT.h +++ b/FWCore/Framework/src/OutputModuleCommunicatorT.h @@ -5,6 +5,7 @@ ----------------------------------------------------------------------*/ #include "FWCore/Framework/src/OutputModuleCommunicator.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" namespace edm { class ActivityRegistry; @@ -65,7 +66,9 @@ namespace edm { edm::SelectedProductsForBranchType const& keptProducts() const override; - void selectProducts(edm::ProductRegistry const& preg, ThinnedAssociationsHelper const&) override; + void selectProducts(edm::ProductRegistry const& preg, + ThinnedAssociationsHelper const&, + ProcessBlockHelperBase const&) override; void setEventSelectionInfo( std::map > > const& outputModulePathPositions, diff --git a/FWCore/Framework/src/Principal.cc b/FWCore/Framework/src/Principal.cc index 47798eabdb057..f249137aa2be1 100644 --- a/FWCore/Framework/src/Principal.cc +++ b/FWCore/Framework/src/Principal.cc @@ -564,6 +564,10 @@ namespace edm { return phb; } + unsigned int Principal::processBlockIndex(std::string const&) const { + throw Exception(errors::LogicError) << "Principal::processBlockIndex not implemented for this type of Principal"; + } + BasicHandle Principal::getByLabel(KindOfType kindOfType, TypeID const& typeID, InputTag const& inputTag, diff --git a/FWCore/Framework/src/PrincipalGetAdapter.cc b/FWCore/Framework/src/PrincipalGetAdapter.cc index 0531a40d3e2ff..93f7e49f05362 100644 --- a/FWCore/Framework/src/PrincipalGetAdapter.cc +++ b/FWCore/Framework/src/PrincipalGetAdapter.cc @@ -85,6 +85,10 @@ namespace edm { consumer_->labelsForToken(iToken, oLabels); } + unsigned int PrincipalGetAdapter::processBlockIndex(std::string const& processName) const { + return principal_.processBlockIndex(processName); + } + BasicHandle PrincipalGetAdapter::makeFailToGetException(KindOfType kindOfType, TypeID const& productType, EDGetToken token) const { diff --git a/FWCore/Framework/src/ProcessBlock.cc b/FWCore/Framework/src/ProcessBlock.cc index b7e3215c4c55c..e98a33e38ca63 100644 --- a/FWCore/Framework/src/ProcessBlock.cc +++ b/FWCore/Framework/src/ProcessBlock.cc @@ -20,6 +20,8 @@ namespace edm { return processBlockPrincipal().cacheIdentifier(); } + std::string const& ProcessBlock::processName() const { return processBlockPrincipal().processName(); } + ProcessBlockPrincipal const& ProcessBlock::processBlockPrincipal() const { return dynamic_cast(provRecorder_.principal()); } diff --git a/FWCore/Framework/src/ProcessBlockForOutput.cc b/FWCore/Framework/src/ProcessBlockForOutput.cc new file mode 100644 index 0000000000000..85e805633fb27 --- /dev/null +++ b/FWCore/Framework/src/ProcessBlockForOutput.cc @@ -0,0 +1,13 @@ +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockPrincipal.h" + +namespace edm { + ProcessBlockForOutput::ProcessBlockForOutput(ProcessBlockPrincipal const& pbp, + ModuleDescription const& md, + ModuleCallingContext const* mcc, + bool isAtEnd) + : OccurrenceForOutput(pbp, md, mcc, isAtEnd), processName_(&pbp.processName()) {} + + ProcessBlockForOutput::~ProcessBlockForOutput() {} + +} // namespace edm diff --git a/FWCore/Framework/src/ProductResolvers.cc b/FWCore/Framework/src/ProductResolvers.cc index 93b7c793aef5e..17832689a4af6 100644 --- a/FWCore/Framework/src/ProductResolvers.cc +++ b/FWCore/Framework/src/ProductResolvers.cc @@ -6,7 +6,6 @@ #include "UnscheduledConfigurator.h" #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/interface/MergeableRunProductMetadata.h" -#include "FWCore/Framework/interface/Principal.h" #include "FWCore/Framework/src/ProductDeletedException.h" #include "FWCore/Framework/interface/SharedResourcesAcquirer.h" #include "FWCore/Framework/interface/DelayedReader.h" @@ -168,12 +167,12 @@ namespace edm { // The file may already be closed so the reader is invalid return; } - if (mcc and (branchType == InEvent || branchType == InProcess) and aux_) { + if (mcc and branchType == InEvent and aux_) { aux_->preModuleDelayedGetSignal_.emit(*(mcc->getStreamContext()), *mcc); } auto sentry(make_sentry(mcc, [this, branchType](ModuleCallingContext const* iContext) { - if ((branchType == InEvent || branchType == InProcess) and aux_) { + if (branchType == InEvent and aux_) { aux_->postModuleDelayedGetSignal_.emit(*(iContext->getStreamContext()), *iContext); } })); @@ -270,7 +269,7 @@ namespace edm { // Caught exception is propagated via WaitingTaskList CMS_SA_ALLOW try { resolveProductImpl([this, &principal, mcc]() { - if (principal.branchType() != InEvent) { + if (principal.branchType() != InEvent && principal.branchType() != InProcess) { return; } if (auto reader = principal.reader()) { @@ -371,11 +370,19 @@ namespace edm { SharedResourcesAcquirer* sra, ModuleCallingContext const* mcc) const { if (not skipCurrentProcess) { + if (branchDescription().branchType() == InProcess && + mcc->parent().globalContext()->transition() == GlobalContext::Transition::kAccessInputProcessBlock) { + // This is an accessInputProcessBlock transition + // We cannot access produced products in those transitions + // except for in SubProcesses where they should have already run. + return; + } if (branchDescription().availableOnlyAtEndTransition() and mcc) { if (not mcc->parent().isAtEndTransition()) { return; } } + //Need to try modifying prefetchRequested_ before adding to m_waitingTasks bool expected = false; bool prefetchRequested = prefetchRequested_.compare_exchange_strong(expected, true); diff --git a/FWCore/Framework/src/ProductResolvers.h b/FWCore/Framework/src/ProductResolvers.h index efec00a8901b3..bd4326d36a121 100644 --- a/FWCore/Framework/src/ProductResolvers.h +++ b/FWCore/Framework/src/ProductResolvers.h @@ -15,24 +15,25 @@ a set of related EDProducts. This is the storage unit of such information. #include "DataFormats/Provenance/interface/BranchDescription.h" #include "DataFormats/Provenance/interface/BranchID.h" #include "DataFormats/Provenance/interface/Provenance.h" +#include "FWCore/Framework/interface/Principal.h" +#include "FWCore/ServiceRegistry/interface/GlobalContext.h" +#include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/ProductResolverIndex.h" #include "FWCore/Utilities/interface/TypeID.h" #include "FWCore/Utilities/interface/thread_safety_macros.h" #include "FWCore/Concurrency/interface/WaitingTaskList.h" #include "FWCore/Concurrency/interface/WaitingTaskHolder.h" -#include #include - +#include #include namespace edm { class MergeableRunProductMetadata; class ProductProvenanceRetriever; class DelayedReader; - class ModuleCallingContext; class SharedResourcesAcquirer; - class Principal; class UnscheduledAuxiliary; class Worker; class ServiceToken; @@ -422,6 +423,12 @@ namespace edm { bool skipCurrentProcess, SharedResourcesAcquirer* sra, ModuleCallingContext const* mcc) const override { + if (principal.branchType() == InProcess && + (mcc->parent().globalContext()->transition() == GlobalContext::Transition::kBeginProcessBlock || + mcc->parent().globalContext()->transition() == GlobalContext::Transition::kEndProcessBlock)) { + return Resolution(nullptr); + } + skipCurrentProcess = false; return realProduct_->resolveProduct(*parentPrincipal_, skipCurrentProcess, sra, mcc); } @@ -431,6 +438,12 @@ namespace edm { ServiceToken const& token, SharedResourcesAcquirer* sra, ModuleCallingContext const* mcc) const override { + if (principal.branchType() == InProcess && + (mcc->parent().globalContext()->transition() == GlobalContext::Transition::kBeginProcessBlock || + mcc->parent().globalContext()->transition() == GlobalContext::Transition::kEndProcessBlock)) { + return; + } + skipCurrentProcess = false; realProduct_->prefetchAsync(waitTask, *parentPrincipal_, skipCurrentProcess, token, sra, mcc); } diff --git a/FWCore/Framework/src/Schedule.cc b/FWCore/Framework/src/Schedule.cc index 41e674fdbec66..d5dc57141d566 100644 --- a/FWCore/Framework/src/Schedule.cc +++ b/FWCore/Framework/src/Schedule.cc @@ -7,6 +7,7 @@ #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "DataFormats/Provenance/interface/BranchIDListHelper.h" #include "DataFormats/Provenance/interface/ProductResolverIndexHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" #include "FWCore/Framework/interface/EDConsumerBase.h" #include "FWCore/Framework/src/OutputModuleDescription.h" #include "FWCore/Framework/src/SubProcess.h" @@ -658,6 +659,7 @@ namespace edm { service::TriggerNamesService const& tns, ProductRegistry& preg, BranchIDListHelper& branchIDListHelper, + ProcessBlockHelperBase& processBlockHelper, ThinnedAssociationsHelper& thinnedAssociationsHelper, SubProcessParentageHelper const* subProcessParentageHelper, ExceptionToActionTable const& actions, @@ -796,10 +798,12 @@ namespace edm { worker->registerThinnedAssociations(preg, thinnedAssociationsHelper); } + processBlockHelper.updateForNewProcess(preg, processConfiguration->processName()); + // The output modules consume products in kept branches. // So we must set this up before freezing. for (auto& c : all_output_communicators_) { - c->selectProducts(preg, thinnedAssociationsHelper); + c->selectProducts(preg, thinnedAssociationsHelper, processBlockHelper); } for (auto& product : preg.productListUpdator()) { @@ -1213,6 +1217,9 @@ namespace edm { void Schedule::closeOutputFiles() { using std::placeholders::_1; for_all(all_output_communicators_, std::bind(&OutputModuleCommunicator::closeFile, _1)); + for (auto& worker : allWorkers()) { + worker->respondToCloseOutputFile(); + } } void Schedule::openOutputFiles(FileBlock& fb) { @@ -1352,8 +1359,10 @@ namespace edm { for_all(allWorkers(), std::bind(&Worker::respondToCloseInputFile, _1, std::cref(fb))); } - void Schedule::beginJob(ProductRegistry const& iRegistry, eventsetup::ESRecordsToProxyIndices const& iESIndices) { - globalSchedule_->beginJob(iRegistry, iESIndices); + void Schedule::beginJob(ProductRegistry const& iRegistry, + eventsetup::ESRecordsToProxyIndices const& iESIndices, + ProcessBlockHelperBase const& processBlockHelperBase) { + globalSchedule_->beginJob(iRegistry, iESIndices, processBlockHelperBase); } void Schedule::beginStream(unsigned int iStreamID) { diff --git a/FWCore/Framework/src/ScheduleItems.cc b/FWCore/Framework/src/ScheduleItems.cc index fbbcb48e5ef0a..5e1bc2348f3a3 100644 --- a/FWCore/Framework/src/ScheduleItems.cc +++ b/FWCore/Framework/src/ScheduleItems.cc @@ -6,6 +6,7 @@ #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "DataFormats/Provenance/interface/SubProcessParentageHelper.h" #include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "FWCore/Common/interface/SubProcessBlockHelper.h" #include "FWCore/Framework/interface/ExceptionActions.h" #include "FWCore/Framework/src/CommonParams.h" #include "FWCore/Framework/interface/ConstProductRegistry.h" @@ -34,7 +35,10 @@ namespace edm { act_table_(), processConfiguration_() {} - ScheduleItems::ScheduleItems(ProductRegistry const& preg, SubProcess const& om) + ScheduleItems::ScheduleItems(ProductRegistry const& preg, + SubProcess const& om, + SubProcessBlockHelper& subProcessBlockHelper, + ProcessBlockHelperBase const& parentProcessBlockHelper) : actReg_(std::make_shared()), preg_(std::make_shared(preg)), branchIDListHelper_(std::make_shared()), @@ -76,6 +80,7 @@ namespace edm { prod.setDropped(true); } } + subProcessBlockHelper.updateFromParentProcess(parentProcessBlockHelper, *preg_); } ServiceToken ScheduleItems::initServices(std::vector& pServiceSets, @@ -130,11 +135,13 @@ namespace edm { std::unique_ptr ScheduleItems::initSchedule(ParameterSet& parameterSet, bool hasSubprocesses, PreallocationConfiguration const& config, - ProcessContext const* processContext) { + ProcessContext const* processContext, + ProcessBlockHelperBase& processBlockHelper) { return std::make_unique(parameterSet, ServiceRegistry::instance().get(), *preg_, *branchIDListHelper_, + processBlockHelper, *thinnedAssociationsHelper_, subProcessParentageHelper_ ? subProcessParentageHelper_.get() : nullptr, *act_table_, diff --git a/FWCore/Framework/src/SubProcess.cc b/FWCore/Framework/src/SubProcess.cc index 3d1e1446b1869..e38a222a37541 100644 --- a/FWCore/Framework/src/SubProcess.cc +++ b/FWCore/Framework/src/SubProcess.cc @@ -10,6 +10,7 @@ #include "DataFormats/Provenance/interface/RunAuxiliary.h" #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "DataFormats/Provenance/interface/SubProcessParentageHelper.h" +#include "FWCore/Common/interface/SubProcessBlockHelper.h" #include "FWCore/Framework/interface/EventForOutput.h" #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/interface/FileBlock.h" @@ -41,6 +42,7 @@ #include "boost/range/adaptor/reversed.hpp" #include +#include #include namespace edm { @@ -49,6 +51,7 @@ namespace edm { ParameterSet const& topLevelParameterSet, std::shared_ptr parentProductRegistry, std::shared_ptr parentBranchIDListHelper, + ProcessBlockHelperBase const& parentProcessBlockHelper, ThinnedAssociationsHelper const& parentThinnedAssociationsHelper, SubProcessParentageHelper const& parentSubProcessParentageHelper, eventsetup::EventSetupsController& esController, @@ -126,7 +129,9 @@ namespace edm { // parameters were not explicitly set. validateTopLevelParameterSets(processParameterSet_.get()); - ScheduleItems items(*parentProductRegistry, *this); + processBlockHelper_ = std::make_shared(); + + ScheduleItems items(*parentProductRegistry, *this, *processBlockHelper_, parentProcessBlockHelper); actReg_ = items.actReg_; //initialize the services @@ -156,7 +161,8 @@ namespace edm { parentThinnedAssociationsHelper, keepAssociation, droppedBranchIDToKeptBranchID_); // intialize the Schedule - schedule_ = items.initSchedule(*processParameterSet_, hasSubProcesses, preallocConfig, &processContext_); + schedule_ = items.initSchedule( + *processParameterSet_, hasSubProcesses, preallocConfig, &processContext_, *processBlockHelper_); // set the items act_table_ = std::move(items.act_table_); @@ -182,7 +188,8 @@ namespace edm { *processConfiguration_, &(historyAppenders_[index]), index, - false /*not primary process*/); + false /*not primary process*/, + &*processBlockHelper_); principalCache_.insert(ep); } for (unsigned int index = 0; index < preallocConfig.numberOfLuminosityBlocks(); ++index) { @@ -207,6 +214,7 @@ namespace edm { topLevelParameterSet, preg_, branchIDListHelper(), + *processBlockHelper_, *thinnedAssociationsHelper_, *subProcessParentageHelper_, esController, @@ -291,7 +299,7 @@ namespace edm { } ServiceRegistry::Operate operate(serviceToken_); actReg_->preBeginJobSignal_(pathsAndConsumesOfModules_, processContext_); - schedule_->beginJob(*preg_, esp_->recordsToProxyIndices()); + schedule_->beginJob(*preg_, esp_->recordsToProxyIndices(), *processBlockHelper_); for_all(subProcesses_, [](auto& subProcess) { subProcess.doBeginJob(); }); } @@ -425,6 +433,7 @@ namespace edm { &principal.processHistory(), std::move(esids), std::move(bli), + principal.eventToProcessBlockIndexes(), *(principal.productProvenanceRetrieverPtr()), //NOTE: this transfers the per product provenance principal.reader(), deepCopyRetriever); @@ -462,7 +471,7 @@ namespace edm { template <> void SubProcess::doBeginProcessBlockAsync>( - WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo) { + WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo, bool cleaningUpAfterException) { ServiceRegistry::Operate operate(serviceToken_); ProcessBlockPrincipal& processBlockPrincipal = principalCache_.inputProcessBlockPrincipal(); @@ -472,12 +481,13 @@ namespace edm { ProcessBlockTransitionInfo transitionInfo(processBlockPrincipal); using Traits = OccurrenceTraits; - beginGlobalTransitionAsync(std::move(iHolder), *schedule_, transitionInfo, serviceToken_, subProcesses_); + beginGlobalTransitionAsync( + std::move(iHolder), *schedule_, transitionInfo, serviceToken_, subProcesses_, cleaningUpAfterException); } template <> void SubProcess::doBeginProcessBlockAsync>( - WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo) { + WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo, bool) { ServiceRegistry::Operate operate(serviceToken_); ProcessBlockPrincipal& processBlockPrincipal = principalCache_.processBlockPrincipal(); @@ -498,9 +508,54 @@ namespace edm { propagateProducts(InProcess, parentPrincipal, processBlockPrincipal); ProcessBlockTransitionInfo transitionInfo(processBlockPrincipal); - using Traits = OccurrenceTraits; - endGlobalTransitionAsync( - std::move(iHolder), *schedule_, transitionInfo, serviceToken_, subProcesses_, cleaningUpAfterException); + + if (parentProducedProductIsKept(parentPrincipal, processBlockPrincipal)) { + auto runEndProcessBlock = + make_waiting_task([this, iWait = std::move(iHolder), info = transitionInfo, cleaningUpAfterException]( + std::exception_ptr const* iPtr) mutable { + ProcessBlockPrincipal& inputProcessBlockPrincipal = principalCache_.inputProcessBlockPrincipal(); + inputProcessBlockPrincipal.clearPrincipal(); + for (auto& s : subProcesses_) { + s.clearProcessBlockPrincipal(ProcessBlockType::Input); + } + if (iPtr) { + iWait.doneWaiting(*iPtr); + } else { + using Traits = OccurrenceTraits; + endGlobalTransitionAsync( + std::move(iWait), *schedule_, info, serviceToken_, subProcesses_, cleaningUpAfterException); + } + }); + WaitingTaskHolder holder(*iHolder.group(), runEndProcessBlock); + + auto runWriteProcessBlock = + make_waiting_task([this, iWait = std::move(holder)](std::exception_ptr const* iPtr) mutable { + if (iPtr) { + iWait.doneWaiting(*iPtr); + } else { + writeProcessBlockAsync(iWait, ProcessBlockType::Input); + } + }); + WaitingTaskHolder writeHolder(*iHolder.group(), runWriteProcessBlock); + + ProcessBlockPrincipal& inputProcessBlockPrincipal = principalCache_.inputProcessBlockPrincipal(); + inputProcessBlockPrincipal.fillProcessBlockPrincipal(parentPrincipal.processName(), parentPrincipal.reader()); + propagateProducts(InProcess, parentPrincipal, inputProcessBlockPrincipal); + ProcessBlockTransitionInfo inputTransitionInfo(inputProcessBlockPrincipal); + + using TraitsInput = OccurrenceTraits; + beginGlobalTransitionAsync(std::move(writeHolder), + *schedule_, + inputTransitionInfo, + serviceToken_, + subProcesses_, + cleaningUpAfterException); + + } else { + using Traits = OccurrenceTraits; + endGlobalTransitionAsync( + std::move(iHolder), *schedule_, transitionInfo, serviceToken_, subProcesses_, cleaningUpAfterException); + } } void SubProcess::doBeginRunAsync(WaitingTaskHolder iHolder, RunTransitionInfo const& iTransitionInfo) { @@ -744,6 +799,24 @@ namespace edm { } } + bool SubProcess::parentProducedProductIsKept(Principal const& parentPrincipal, Principal& principal) const { + SelectedProducts const& keptVector = keptProducts()[InProcess]; + for (auto const& item : keptVector) { + BranchDescription const& desc = *item.first; + assert(desc.branchType() == InProcess); + ProductResolverBase const* parentProductResolver = parentPrincipal.getProductResolver(desc.branchID()); + if (parentProductResolver != nullptr) { + ProductResolverBase* productResolver = principal.getModifiableProductResolver(desc.branchID()); + if (productResolver != nullptr) { + if (parentProductResolver->branchDescription().produced()) { + return true; + } + } + } + } + return false; + } + void SubProcess::updateBranchIDListHelper(BranchIDLists const& branchIDLists) { branchIDListHelper_->updateFromParent(branchIDLists); for_all(subProcesses_, diff --git a/FWCore/Framework/src/SubProcess.h b/FWCore/Framework/src/SubProcess.h index ac82e3ecfebaa..fcd3e811508f8 100644 --- a/FWCore/Framework/src/SubProcess.h +++ b/FWCore/Framework/src/SubProcess.h @@ -2,6 +2,7 @@ #define FWCore_Framework_SubProcess_h #include "DataFormats/Provenance/interface/BranchID.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/EventSetupProvider.h" #include "FWCore/Framework/interface/EDConsumerBase.h" #include "FWCore/Framework/interface/PathsAndConsumesOfModules.h" @@ -16,6 +17,7 @@ #include "FWCore/Utilities/interface/Algorithms.h" #include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" #include "DataFormats/Provenance/interface/SelectedProducts.h" @@ -53,6 +55,7 @@ namespace edm { ParameterSet const& topLevelParameterSet, std::shared_ptr parentProductRegistry, std::shared_ptr parentBranchIDListHelper, + ProcessBlockHelperBase const& parentProcessBlockHelper, ThinnedAssociationsHelper const& parentThinnedAssociationsHelper, SubProcessParentageHelper const& parentSubProcessParentageHelper, eventsetup::EventSetupsController& esController, @@ -88,7 +91,9 @@ namespace edm { std::vector> const*); template - void doBeginProcessBlockAsync(WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo); + void doBeginProcessBlockAsync(WaitingTaskHolder iHolder, + ProcessBlockTransitionInfo const& iTransitionInfo, + bool cleaningUpAfterException); void doEndProcessBlockAsync(WaitingTaskHolder iHolder, ProcessBlockTransitionInfo const& iTransitionInfo, @@ -254,6 +259,7 @@ namespace edm { std::vector> const*); void propagateProducts(BranchType type, Principal const& parentPrincipal, Principal& principal) const; + bool parentProducedProductIsKept(Principal const& parentPrincipal, Principal& principal) const; void fixBranchIDListsForEDAliases( std::map const& droppedBranchIDToKeptBranchID); void keepThisBranch(BranchDescription const& desc, @@ -280,6 +286,7 @@ namespace edm { std::shared_ptr parentPreg_; std::shared_ptr preg_; edm::propagate_const> branchIDListHelper_; + edm::propagate_const> processBlockHelper_; edm::propagate_const> thinnedAssociationsHelper_; edm::propagate_const> subProcessParentageHelper_; std::unique_ptr act_table_; diff --git a/FWCore/Framework/src/Worker.h b/FWCore/Framework/src/Worker.h index bc6de5b065112..abd41a7922f1a 100644 --- a/FWCore/Framework/src/Worker.h +++ b/FWCore/Framework/src/Worker.h @@ -22,6 +22,7 @@ the worker is reset(). ----------------------------------------------------------------------*/ #include "DataFormats/Provenance/interface/ModuleDescription.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/MessageLogger/interface/ExceptionMessages.h" #include "FWCore/Framework/src/TransitionInfoTypes.h" #include "FWCore/Framework/src/WorkerParams.h" @@ -170,7 +171,7 @@ namespace edm { void endStream(StreamID id, StreamContext& streamContext); void respondToOpenInputFile(FileBlock const& fb) { implRespondToOpenInputFile(fb); } void respondToCloseInputFile(FileBlock const& fb) { implRespondToCloseInputFile(fb); } - + void respondToCloseOutputFile() { implRespondToCloseOutputFile(); } void registerThinnedAssociations(ProductRegistry const& registry, ThinnedAssociationsHelper& helper); void reset() { @@ -198,6 +199,7 @@ namespace edm { //Used to make EDGetToken work virtual void updateLookup(BranchType iBranchType, ProductResolverIndexHelper const&) = 0; virtual void updateLookup(eventsetup::ESRecordsToProxyIndices const&) = 0; + virtual void selectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&) = 0; virtual void resolvePutIndicies( BranchType iBranchType, std::unordered_multimap> const& @@ -292,6 +294,7 @@ namespace edm { virtual void implRespondToOpenInputFile(FileBlock const& fb) = 0; virtual void implRespondToCloseInputFile(FileBlock const& fb) = 0; + virtual void implRespondToCloseOutputFile() = 0; virtual void implRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) = 0; diff --git a/FWCore/Framework/src/WorkerManager.cc b/FWCore/Framework/src/WorkerManager.cc index 2932813deb00d..2460a194c7def 100644 --- a/FWCore/Framework/src/WorkerManager.cc +++ b/FWCore/Framework/src/WorkerManager.cc @@ -89,7 +89,8 @@ namespace edm { } void WorkerManager::beginJob(ProductRegistry const& iRegistry, - eventsetup::ESRecordsToProxyIndices const& iESIndices) { + eventsetup::ESRecordsToProxyIndices const& iESIndices, + ProcessBlockHelperBase const& processBlockHelperBase) { auto const processBlockLookup = iRegistry.productLookup(InProcess); auto const runLookup = iRegistry.productLookup(InRun); auto const lumiLookup = iRegistry.productLookup(InLumi); @@ -110,6 +111,7 @@ namespace edm { worker->resolvePutIndicies(InRun, runModuleToIndicies); worker->resolvePutIndicies(InLumi, lumiModuleToIndicies); worker->resolvePutIndicies(InEvent, eventModuleToIndicies); + worker->selectInputProcessBlocks(iRegistry, processBlockHelperBase); } for_all(allWorkers_, std::bind(&Worker::beginJob, std::placeholders::_1)); diff --git a/FWCore/Framework/src/WorkerT.cc b/FWCore/Framework/src/WorkerT.cc index 3978441cce32c..a693c6071b8b0 100644 --- a/FWCore/Framework/src/WorkerT.cc +++ b/FWCore/Framework/src/WorkerT.cc @@ -484,6 +484,11 @@ namespace edm { module_->doRespondToCloseInputFile(fb); } + template + void WorkerT::implRespondToCloseOutputFile() { + module_->doRespondToCloseOutputFile(); + } + template inline void WorkerT::implRegisterThinnedAssociations(ProductRegistry const& registry, ThinnedAssociationsHelper& helper) { @@ -632,6 +637,12 @@ namespace edm { module_->updateLookup(iPI); } + template + void WorkerT::selectInputProcessBlocks(ProductRegistry const& productRegistry, + ProcessBlockHelperBase const& processBlockHelperBase) { + module_->selectInputProcessBlocks(productRegistry, processBlockHelperBase); + } + namespace { using ModuleToResolverIndicies = std::unordered_multimap>; diff --git a/FWCore/Framework/src/WorkerT.h b/FWCore/Framework/src/WorkerT.h index e373ee736eca4..d42930c14a91c 100644 --- a/FWCore/Framework/src/WorkerT.h +++ b/FWCore/Framework/src/WorkerT.h @@ -7,6 +7,7 @@ WorkerT: Code common to all workers. ----------------------------------------------------------------------*/ +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/src/TransitionInfoTypes.h" #include "FWCore/Framework/src/Worker.h" @@ -55,6 +56,8 @@ namespace edm { void updateLookup(BranchType iBranchType, ProductResolverIndexHelper const&) final; void updateLookup(eventsetup::ESRecordsToProxyIndices const&) final; + void selectInputProcessBlocks(ProductRegistry const&, ProcessBlockHelperBase const&) final; + void resolvePutIndicies( BranchType iBranchType, std::unordered_multimap> const& @@ -105,6 +108,7 @@ namespace edm { void implEndStream(StreamID) override; void implRespondToOpenInputFile(FileBlock const& fb) override; void implRespondToCloseInputFile(FileBlock const& fb) override; + void implRespondToCloseOutputFile() override; void implRegisterThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) override; std::string workerType() const override; TaskQueueAdaptor serializeRunModule() override; diff --git a/FWCore/Framework/src/global/EDAnalyzerBase.cc b/FWCore/Framework/src/global/EDAnalyzerBase.cc index 714deba33f840..64ca4ad3fce9a 100644 --- a/FWCore/Framework/src/global/EDAnalyzerBase.cc +++ b/FWCore/Framework/src/global/EDAnalyzerBase.cc @@ -198,14 +198,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDAnalyzerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDAnalyzerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDAnalyzerBase::preallocStreams(unsigned int) {} void EDAnalyzerBase::preallocLumis(unsigned int) {} void EDAnalyzerBase::preallocLumisSummary(unsigned int) {} @@ -233,6 +225,8 @@ namespace edm { void EDAnalyzerBase::doBeginLuminosityBlockSummary_(LuminosityBlock const& rp, EventSetup const& c) {} void EDAnalyzerBase::doEndLuminosityBlockSummary_(LuminosityBlock const& lb, EventSetup const& c) {} + void EDAnalyzerBase::clearInputProcessBlockCaches() {} + void EDAnalyzerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/global/EDFilterBase.cc b/FWCore/Framework/src/global/EDFilterBase.cc index ac3e457ec1665..488d8799c37c0 100644 --- a/FWCore/Framework/src/global/EDFilterBase.cc +++ b/FWCore/Framework/src/global/EDFilterBase.cc @@ -243,14 +243,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDFilterBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDFilterBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDFilterBase::preallocStreams(unsigned int) {} void EDFilterBase::preallocLumis(unsigned int) {} void EDFilterBase::preallocLumisSummary(unsigned int) {} @@ -286,6 +278,8 @@ namespace edm { void EDFilterBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDFilterBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDFilterBase::clearInputProcessBlockCaches() {} + void EDFilterBase::doAcquire_(StreamID, Event const&, EventSetup const&, WaitingTaskWithArenaHolder&) {} void EDFilterBase::fillDescriptions(ConfigurationDescriptions& descriptions) { diff --git a/FWCore/Framework/src/global/EDProducerBase.cc b/FWCore/Framework/src/global/EDProducerBase.cc index 6b5ef9219aec4..8d24bcb498b70 100644 --- a/FWCore/Framework/src/global/EDProducerBase.cc +++ b/FWCore/Framework/src/global/EDProducerBase.cc @@ -252,14 +252,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDProducerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDProducerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDProducerBase::preallocStreams(unsigned int) {} void EDProducerBase::preallocLumis(unsigned int) {} void EDProducerBase::preallocLumisSummary(unsigned int) {} @@ -295,6 +287,8 @@ namespace edm { void EDProducerBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDProducerBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDProducerBase::clearInputProcessBlockCaches() {} + void EDProducerBase::doAcquire_(StreamID, Event const&, EventSetup const&, WaitingTaskWithArenaHolder&) {} void EDProducerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { diff --git a/FWCore/Framework/src/global/OutputModuleBase.cc b/FWCore/Framework/src/global/OutputModuleBase.cc index d9310aa613c0b..5ab7c2a2aec0e 100644 --- a/FWCore/Framework/src/global/OutputModuleBase.cc +++ b/FWCore/Framework/src/global/OutputModuleBase.cc @@ -24,6 +24,7 @@ #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/src/insertSelectedProcesses.h" #include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" #include "FWCore/Framework/interface/RunForOutput.h" #include "FWCore/Framework/src/OutputModuleDescription.h" #include "FWCore/Framework/interface/TriggerNamesService.h" @@ -80,7 +81,8 @@ namespace edm { } void OutputModuleBase::selectProducts(ProductRegistry const& preg, - ThinnedAssociationsHelper const& thinnedAssociationsHelper) { + ThinnedAssociationsHelper const& thinnedAssociationsHelper, + ProcessBlockHelperBase const& processBlockHelper) { if (productSelector_.initialized()) return; productSelector_.initialize(productSelectorRules_, preg.allBranchDescriptions()); @@ -93,6 +95,7 @@ namespace edm { std::vector associationDescriptions; std::set keptProductsInEvent; std::set processesWithSelectedMergeableRunProducts; + std::set processesWithKeptProcessBlockProducts; for (auto const& it : preg.productList()) { BranchDescription const& desc = it.second; @@ -105,7 +108,8 @@ namespace edm { associationDescriptions.push_back(&desc); } else if (selected(desc)) { keepThisBranch(desc, trueBranchIDToKeptBranchDesc, keptProductsInEvent); - insertSelectedProcesses(desc, processesWithSelectedMergeableRunProducts); + insertSelectedProcesses( + desc, processesWithSelectedMergeableRunProducts, processesWithKeptProcessBlockProducts); } else { // otherwise, output nothing, // and mark the fact that there is a newly dropped branch of this type. @@ -131,6 +135,7 @@ namespace edm { thinnedAssociationsHelper_->updateFromParentProcess( thinnedAssociationsHelper, keepAssociation_, droppedBranchIDToKeptBranchID_); + outputProcessBlockHelper_.updateAfterProductSelection(processesWithKeptProcessBlockProducts, processBlockHelper); } void OutputModuleBase::updateBranchIDListsWithKeptAliases() { @@ -285,6 +290,12 @@ namespace edm { return true; } + void OutputModuleBase::doWriteProcessBlock(ProcessBlockPrincipal const& pbp, ModuleCallingContext const* mcc) { + ProcessBlockForOutput pb(pbp, moduleDescription_, mcc, true); + pb.setConsumer(this); + writeProcessBlock(pb); + } + void OutputModuleBase::doWriteRun(RunPrincipal const& rp, ModuleCallingContext const* mcc, MergeableRunProductMetadata const* mrpm) { diff --git a/FWCore/Framework/src/globalTransitionAsync.h b/FWCore/Framework/src/globalTransitionAsync.h index e9b6ec9f56fc2..b0ee00f7a6c96 100644 --- a/FWCore/Framework/src/globalTransitionAsync.h +++ b/FWCore/Framework/src/globalTransitionAsync.h @@ -34,22 +34,25 @@ namespace edm { template inline void subProcessDoGlobalBeginTransitionAsync(WaitingTaskHolder iHolder, SubProcess& iSubProcess, - LumiTransitionInfo const& iTransitionInfo) { + LumiTransitionInfo const& iTransitionInfo, + bool) { iSubProcess.doBeginLuminosityBlockAsync(std::move(iHolder), iTransitionInfo); } template inline void subProcessDoGlobalBeginTransitionAsync(WaitingTaskHolder iHolder, SubProcess& iSubProcess, - RunTransitionInfo const& iTransitionInfo) { + RunTransitionInfo const& iTransitionInfo, + bool) { iSubProcess.doBeginRunAsync(std::move(iHolder), iTransitionInfo); } template inline void subProcessDoGlobalBeginTransitionAsync(WaitingTaskHolder iHolder, SubProcess& iSubProcess, - ProcessBlockTransitionInfo const& iTransitionInfo) { - iSubProcess.doBeginProcessBlockAsync(std::move(iHolder), iTransitionInfo); + ProcessBlockTransitionInfo const& iTransitionInfo, + bool cleaningUpAfterException) { + iSubProcess.doBeginProcessBlockAsync(std::move(iHolder), iTransitionInfo, cleaningUpAfterException); } inline void subProcessDoGlobalEndTransitionAsync(WaitingTaskHolder iHolder, @@ -78,28 +81,29 @@ namespace edm { Schedule& iSchedule, typename Traits::TransitionInfoType& transitionInfo, ServiceToken const& token, - std::vector& iSubProcesses) { + std::vector& iSubProcesses, + bool cleaningUpAfterException = false) { // When we are done processing the global for this process, // we need to run the global for all SubProcesses - auto subs = - make_waiting_task([&iSubProcesses, iWait, info = transitionInfo](std::exception_ptr const* iPtr) mutable { - if (iPtr) { - auto excpt = *iPtr; - auto delayError = - make_waiting_task([iWait, excpt](std::exception_ptr const*) mutable { iWait.doneWaiting(excpt); }); - WaitingTaskHolder h(*iWait.group(), delayError); - for (auto& subProcess : iSubProcesses) { - subProcessDoGlobalBeginTransitionAsync(h, subProcess, info); - } - } else { - for (auto& subProcess : iSubProcesses) { - subProcessDoGlobalBeginTransitionAsync(iWait, subProcess, info); - } - } - }); + auto subs = make_waiting_task([&iSubProcesses, iWait, info = transitionInfo, cleaningUpAfterException]( + std::exception_ptr const* iPtr) mutable { + if (iPtr) { + auto excpt = *iPtr; + auto delayError = + make_waiting_task([iWait, excpt](std::exception_ptr const*) mutable { iWait.doneWaiting(excpt); }); + WaitingTaskHolder h(*iWait.group(), delayError); + for (auto& subProcess : iSubProcesses) { + subProcessDoGlobalBeginTransitionAsync(h, subProcess, info, cleaningUpAfterException); + } + } else { + for (auto& subProcess : iSubProcesses) { + subProcessDoGlobalBeginTransitionAsync(iWait, subProcess, info, cleaningUpAfterException); + } + } + }); WaitingTaskHolder h(*iWait.group(), subs); - iSchedule.processOneGlobalAsync(std::move(h), transitionInfo, token); + iSchedule.processOneGlobalAsync(std::move(h), transitionInfo, token, cleaningUpAfterException); } template diff --git a/FWCore/Framework/src/insertSelectedProcesses.cc b/FWCore/Framework/src/insertSelectedProcesses.cc index 413128c679a59..0bc25157d2017 100644 --- a/FWCore/Framework/src/insertSelectedProcesses.cc +++ b/FWCore/Framework/src/insertSelectedProcesses.cc @@ -12,7 +12,9 @@ namespace edm { - void insertSelectedProcesses(BranchDescription const& desc, std::set& processes) { + void insertSelectedProcesses(BranchDescription const& desc, + std::set& processes, + std::set& processesWithKeptProcessBlockProducts) { // Select input processes in which mergeable run products were produced if (desc.branchType() == InRun && !desc.produced()) { // Determine if the product is "mergeable" @@ -26,5 +28,8 @@ namespace edm { processes.insert(desc.processName()); } } + if (desc.branchType() == InProcess) { + processesWithKeptProcessBlockProducts.insert(desc.processName()); + } } } // namespace edm diff --git a/FWCore/Framework/src/insertSelectedProcesses.h b/FWCore/Framework/src/insertSelectedProcesses.h index 501b0dbad8a1d..e31082dda3fe9 100644 --- a/FWCore/Framework/src/insertSelectedProcesses.h +++ b/FWCore/Framework/src/insertSelectedProcesses.h @@ -8,6 +8,8 @@ namespace edm { class BranchDescription; - void insertSelectedProcesses(BranchDescription const& desc, std::set& processes); + void insertSelectedProcesses(BranchDescription const& desc, + std::set& processes, + std::set& processesWithKeptProcessBlockProducts); } // namespace edm #endif diff --git a/FWCore/Framework/src/limited/EDAnalyzerBase.cc b/FWCore/Framework/src/limited/EDAnalyzerBase.cc index 3a5f52c39abe6..af70265548374 100644 --- a/FWCore/Framework/src/limited/EDAnalyzerBase.cc +++ b/FWCore/Framework/src/limited/EDAnalyzerBase.cc @@ -199,14 +199,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDAnalyzerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDAnalyzerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDAnalyzerBase::preallocStreams(unsigned int) {} void EDAnalyzerBase::preallocLumis(unsigned int) {} void EDAnalyzerBase::preallocLumisSummary(unsigned int) {} @@ -235,6 +227,8 @@ namespace edm { void EDAnalyzerBase::doBeginLuminosityBlockSummary_(LuminosityBlock const& rp, EventSetup const& c) {} void EDAnalyzerBase::doEndLuminosityBlockSummary_(LuminosityBlock const& lb, EventSetup const& c) {} + void EDAnalyzerBase::clearInputProcessBlockCaches() {} + void EDAnalyzerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/limited/EDFilterBase.cc b/FWCore/Framework/src/limited/EDFilterBase.cc index 436b5eaf512af..2f26a2e03f8a4 100644 --- a/FWCore/Framework/src/limited/EDFilterBase.cc +++ b/FWCore/Framework/src/limited/EDFilterBase.cc @@ -224,14 +224,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDFilterBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDFilterBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDFilterBase::preallocStreams(unsigned int) {} void EDFilterBase::preallocLumis(unsigned int) {} void EDFilterBase::preallocLumisSummary(unsigned int) {} @@ -267,6 +259,8 @@ namespace edm { void EDFilterBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDFilterBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDFilterBase::clearInputProcessBlockCaches() {} + void EDFilterBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/limited/EDProducerBase.cc b/FWCore/Framework/src/limited/EDProducerBase.cc index 022457ff9ea2f..8d167ec15ea73 100644 --- a/FWCore/Framework/src/limited/EDProducerBase.cc +++ b/FWCore/Framework/src/limited/EDProducerBase.cc @@ -224,14 +224,6 @@ namespace edm { this->doStreamEndLuminosityBlockSummary_(id, lb, c); } - void EDProducerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDProducerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDProducerBase::preallocStreams(unsigned int) {} void EDProducerBase::preallocLumis(unsigned int) {} void EDProducerBase::preallocLumisSummary(unsigned int) {} @@ -267,6 +259,8 @@ namespace edm { void EDProducerBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDProducerBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDProducerBase::clearInputProcessBlockCaches() {} + void EDProducerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/limited/OutputModuleBase.cc b/FWCore/Framework/src/limited/OutputModuleBase.cc index 294aca56c6e7c..67fcd9916ec72 100644 --- a/FWCore/Framework/src/limited/OutputModuleBase.cc +++ b/FWCore/Framework/src/limited/OutputModuleBase.cc @@ -24,6 +24,7 @@ #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/src/insertSelectedProcesses.h" #include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" #include "FWCore/Framework/interface/RunForOutput.h" #include "FWCore/Framework/src/OutputModuleDescription.h" #include "FWCore/Framework/interface/TriggerNamesService.h" @@ -81,7 +82,8 @@ namespace edm { } void OutputModuleBase::selectProducts(ProductRegistry const& preg, - ThinnedAssociationsHelper const& thinnedAssociationsHelper) { + ThinnedAssociationsHelper const& thinnedAssociationsHelper, + ProcessBlockHelperBase const& processBlockHelper) { if (productSelector_.initialized()) return; productSelector_.initialize(productSelectorRules_, preg.allBranchDescriptions()); @@ -94,6 +96,7 @@ namespace edm { std::vector associationDescriptions; std::set keptProductsInEvent; std::set processesWithSelectedMergeableRunProducts; + std::set processesWithKeptProcessBlockProducts; for (auto const& it : preg.productList()) { BranchDescription const& desc = it.second; @@ -106,7 +109,8 @@ namespace edm { associationDescriptions.push_back(&desc); } else if (selected(desc)) { keepThisBranch(desc, trueBranchIDToKeptBranchDesc, keptProductsInEvent); - insertSelectedProcesses(desc, processesWithSelectedMergeableRunProducts); + insertSelectedProcesses( + desc, processesWithSelectedMergeableRunProducts, processesWithKeptProcessBlockProducts); } else { // otherwise, output nothing, // and mark the fact that there is a newly dropped branch of this type. @@ -132,6 +136,7 @@ namespace edm { thinnedAssociationsHelper_->updateFromParentProcess( thinnedAssociationsHelper, keepAssociation_, droppedBranchIDToKeptBranchID_); + outputProcessBlockHelper_.updateAfterProductSelection(processesWithKeptProcessBlockProducts, processBlockHelper); } void OutputModuleBase::updateBranchIDListsWithKeptAliases() { @@ -286,6 +291,12 @@ namespace edm { return true; } + void OutputModuleBase::doWriteProcessBlock(ProcessBlockPrincipal const& pbp, ModuleCallingContext const* mcc) { + ProcessBlockForOutput pb(pbp, moduleDescription_, mcc, true); + pb.setConsumer(this); + writeProcessBlock(pb); + } + void OutputModuleBase::doWriteRun(RunPrincipal const& rp, ModuleCallingContext const* mcc, MergeableRunProductMetadata const* mrpm) { diff --git a/FWCore/Framework/src/one/EDAnalyzerBase.cc b/FWCore/Framework/src/one/EDAnalyzerBase.cc index e7fea6fd340b2..30518265e1382 100644 --- a/FWCore/Framework/src/one/EDAnalyzerBase.cc +++ b/FWCore/Framework/src/one/EDAnalyzerBase.cc @@ -156,14 +156,6 @@ namespace edm { this->doEndLuminosityBlock_(cnstLb, c); } - void EDAnalyzerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDAnalyzerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDAnalyzerBase::doBeginProcessBlock_(ProcessBlock const&) {} void EDAnalyzerBase::doAccessInputProcessBlock_(ProcessBlock const&) {} void EDAnalyzerBase::doEndProcessBlock_(ProcessBlock const&) {} @@ -173,6 +165,8 @@ namespace edm { void EDAnalyzerBase::doBeginLuminosityBlock_(LuminosityBlock const& lbp, EventSetup const& c) {} void EDAnalyzerBase::doEndLuminosityBlock_(LuminosityBlock const& lbp, EventSetup const& c) {} + void EDAnalyzerBase::clearInputProcessBlockCaches() {} + void EDAnalyzerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/one/EDFilterBase.cc b/FWCore/Framework/src/one/EDFilterBase.cc index 176f2ec7e87aa..60395400dacb2 100644 --- a/FWCore/Framework/src/one/EDFilterBase.cc +++ b/FWCore/Framework/src/one/EDFilterBase.cc @@ -174,14 +174,6 @@ namespace edm { commit_(lb); } - void EDFilterBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDFilterBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDFilterBase::doBeginProcessBlock_(ProcessBlock const&) {} void EDFilterBase::doAccessInputProcessBlock_(ProcessBlock const&) {} void EDFilterBase::doEndProcessBlock_(ProcessBlock const&) {} @@ -197,6 +189,8 @@ namespace edm { void EDFilterBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDFilterBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDFilterBase::clearInputProcessBlockCaches() {} + void EDFilterBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/one/EDProducerBase.cc b/FWCore/Framework/src/one/EDProducerBase.cc index ef1ff4b47d5e3..8b8e369c87d47 100644 --- a/FWCore/Framework/src/one/EDProducerBase.cc +++ b/FWCore/Framework/src/one/EDProducerBase.cc @@ -174,14 +174,6 @@ namespace edm { commit_(lb); } - void EDProducerBase::doRespondToOpenInputFile(FileBlock const& fb) { - //respondToOpenInputFile(fb); - } - - void EDProducerBase::doRespondToCloseInputFile(FileBlock const& fb) { - //respondToCloseInputFile(fb); - } - void EDProducerBase::doBeginProcessBlock_(ProcessBlock const&) {} void EDProducerBase::doAccessInputProcessBlock_(ProcessBlock const&) {} void EDProducerBase::doEndProcessBlock_(ProcessBlock const&) {} @@ -197,6 +189,8 @@ namespace edm { void EDProducerBase::doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} void EDProducerBase::doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c) {} + void EDProducerBase::clearInputProcessBlockCaches() {} + void EDProducerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; desc.setUnknown(); diff --git a/FWCore/Framework/src/one/OutputModuleBase.cc b/FWCore/Framework/src/one/OutputModuleBase.cc index b2e6ba0b1f31e..87a879f1aa983 100644 --- a/FWCore/Framework/src/one/OutputModuleBase.cc +++ b/FWCore/Framework/src/one/OutputModuleBase.cc @@ -26,6 +26,7 @@ #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/src/insertSelectedProcesses.h" #include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" #include "FWCore/Framework/interface/RunForOutput.h" #include "FWCore/Framework/src/OutputModuleDescription.h" #include "FWCore/Framework/interface/TriggerNamesService.h" @@ -83,7 +84,8 @@ namespace edm { } void OutputModuleBase::selectProducts(ProductRegistry const& preg, - ThinnedAssociationsHelper const& thinnedAssociationsHelper) { + ThinnedAssociationsHelper const& thinnedAssociationsHelper, + ProcessBlockHelperBase const& processBlockHelper) { if (productSelector_.initialized()) return; productSelector_.initialize(productSelectorRules_, preg.allBranchDescriptions()); @@ -96,6 +98,7 @@ namespace edm { std::vector associationDescriptions; std::set keptProductsInEvent; std::set processesWithSelectedMergeableRunProducts; + std::set processesWithKeptProcessBlockProducts; for (auto const& it : preg.productList()) { BranchDescription const& desc = it.second; @@ -108,7 +111,8 @@ namespace edm { associationDescriptions.push_back(&desc); } else if (selected(desc)) { keepThisBranch(desc, trueBranchIDToKeptBranchDesc, keptProductsInEvent); - insertSelectedProcesses(desc, processesWithSelectedMergeableRunProducts); + insertSelectedProcesses( + desc, processesWithSelectedMergeableRunProducts, processesWithKeptProcessBlockProducts); } else { // otherwise, output nothing, // and mark the fact that there is a newly dropped branch of this type. @@ -134,6 +138,7 @@ namespace edm { thinnedAssociationsHelper_->updateFromParentProcess( thinnedAssociationsHelper, keepAssociation_, droppedBranchIDToKeptBranchID_); + outputProcessBlockHelper_.updateAfterProductSelection(processesWithKeptProcessBlockProducts, processBlockHelper); } void OutputModuleBase::keepThisBranch(BranchDescription const& desc, @@ -269,6 +274,12 @@ namespace edm { return true; } + void OutputModuleBase::doWriteProcessBlock(ProcessBlockPrincipal const& pbp, ModuleCallingContext const* mcc) { + ProcessBlockForOutput pb(pbp, moduleDescription_, mcc, true); + pb.setConsumer(this); + writeProcessBlock(pb); + } + void OutputModuleBase::doWriteRun(RunPrincipal const& rp, ModuleCallingContext const* mcc, MergeableRunProductMetadata const* mrpm) { diff --git a/FWCore/Framework/src/processBlockUtilities.cc b/FWCore/Framework/src/processBlockUtilities.cc new file mode 100644 index 0000000000000..4a926b2063cc1 --- /dev/null +++ b/FWCore/Framework/src/processBlockUtilities.cc @@ -0,0 +1,11 @@ +#include "FWCore/Framework/interface/processBlockUtilities.h" + +#include "FWCore/Framework/interface/Event.h" + +namespace edm { + + unsigned int eventProcessBlockIndex(Event const& event, std::string const& processName) { + return event.processBlockIndex(processName); + } + +} // namespace edm diff --git a/FWCore/Framework/src/stream/EDAnalyzerAdaptorBase.cc b/FWCore/Framework/src/stream/EDAnalyzerAdaptorBase.cc index 9943544ef6bdd..87b0da0833f4d 100644 --- a/FWCore/Framework/src/stream/EDAnalyzerAdaptorBase.cc +++ b/FWCore/Framework/src/stream/EDAnalyzerAdaptorBase.cc @@ -227,9 +227,6 @@ void EDAnalyzerAdaptorBase::doStreamEndLuminosityBlock(StreamID id, streamEndLuminosityBlockSummary(mod, lb, c); } -void EDAnalyzerAdaptorBase::doRespondToOpenInputFile(FileBlock const&) {} -void EDAnalyzerAdaptorBase::doRespondToCloseInputFile(FileBlock const&) {} - void EDAnalyzerAdaptorBase::setModuleDescriptionPtr(EDAnalyzerBase* m) { m->setModuleDescriptionPtr(&moduleDescription_); } diff --git a/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc b/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc index 5ed6c8bc0c5b4..39ee2326670f2 100644 --- a/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc +++ b/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc @@ -253,11 +253,6 @@ namespace edm { streamEndLuminosityBlockSummary(mod, lb, c); } - template - void ProducingModuleAdaptorBase::doRespondToOpenInputFile(FileBlock const&) {} - template - void ProducingModuleAdaptorBase::doRespondToCloseInputFile(FileBlock const&) {} - template void ProducingModuleAdaptorBase::doRegisterThinnedAssociations(ProductRegistry const& registry, ThinnedAssociationsHelper& helper) { diff --git a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc index 06594397a930f..3c68948e3c0f4 100644 --- a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc @@ -6,25 +6,28 @@ edm::*Cache templates for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/global/EDAnalyzer.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/global/EDAnalyzer.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" -#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace global { @@ -343,7 +346,7 @@ namespace edmtest { return std::make_unique(); } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 5) { throw cms::Exception("transitions") << "ProcessBlockIntAnalyzer::beginProcessBlock transition " << m_count << " but it was supposed to be " << 5; @@ -383,7 +386,7 @@ namespace edmtest { ++m_count; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 646u) { throw cms::Exception("transitions") << "ProcessBlockIntAnalyzer::endProcessBlock transition " << m_count << " but it was supposed to be " << 646; @@ -439,6 +442,381 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntAnalyzer + : public edm::global::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntAnalyzer(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntAnalyzer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + + class InputProcessBlockAnalyzerThreeTags + : public edm::global::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockAnalyzerThreeTags(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun0_ = pset.getParameter>("expectedByRun0"); + expectedByRun1_ = pset.getParameter>("expectedByRun1"); + expectedByRun2_ = pset.getParameter>("expectedByRun2"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock0"); + if (not tag.label().empty()) { + getTokenBegin0_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock0"); + if (not tag.label().empty()) { + getTokenEnd0_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlock1"); + if (not tag.label().empty()) { + getTokenBegin1_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock1"); + if (not tag.label().empty()) { + getTokenEnd1_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlock2"); + if (not tag.label().empty()) { + getTokenBegin2_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock2"); + if (not tag.label().empty()) { + getTokenEnd2_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin0_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin0_).value; + *returnValue += processBlock.get(getTokenEnd0_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin1_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin1_).value; + returnValue->value_ += processBlock.get(getTokenEnd1_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin2_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin2_).value; + returnValue->value_ += processBlock.get(getTokenEnd2_).value; + ++transitions_; + return returnValue; + }); + } + + void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (expectedByRun0_.empty()) { + if (std::get>(cacheTuple).isValid()) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 0"; + } + } else { + if (expectedByRun0_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze zeroth cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun0_[event.run()]; + } + } + if (expectedByRun1_.empty()) { + if (std::get<1>(cacheTuple).isValid()) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 1"; + } + } else { + if (expectedByRun1_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze first cached value was " + << std::get<1>(cacheTuple)->value_ << " but it was supposed to be " << expectedByRun1_[event.run()]; + } + } + if (expectedByRun2_.empty()) { + if (std::get>(cacheTuple).isValid()) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 2"; + } + } else { + if (expectedByRun2_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags::analyze second cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun2_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockAnalyzerThreeTags transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerThreeTags cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin0_; + edm::EDGetTokenT getTokenEnd0_; + edm::EDGetTokenT getTokenBegin1_; + edm::EDGetTokenT getTokenEnd1_; + edm::EDGetTokenT getTokenBegin2_; + edm::EDGetTokenT getTokenEnd2_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun0_; + std::vector expectedByRun1_; + std::vector expectedByRun2_; + }; + + class InputProcessBlockAnalyzerReuseCache + : public edm::global::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockAnalyzerReuseCache(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + ++transitions_; + auto returnValue = std::make_shared(); + if (previousCache) { + returnValue = previousCache; + return returnValue; + } + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + return returnValue; + }); + } + + void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerReuseCache::analyze cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockAnalyzerReuseCache transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockAnalyzerReuseCache cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + }; + + class InputProcessBlockIntAnalyzerNoRegistration + : public edm::global::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntAnalyzerNoRegistration(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + } + + void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + ++transitions_; + if (std::get<0>(cacheTuple).isValid() || std::get<1>(cacheTuple).isValid() || + std::get<2>(cacheTuple).isValid()) { + throw cms::Exception("LogicError") + << "InputProcessBlockIntAnalyzerNoRegistration expected cacheTuple full of invalid CacheHandles"; + } + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntAnalyzerNoRegistration transitions " + << transitions_ << " but it was supposed to be " << expectedTransitions_; + } + } + + private: + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + unsigned int expectedTransitions_{0}; + }; + } // namespace global } // namespace edmtest @@ -448,3 +826,7 @@ DEFINE_FWK_MODULE(edmtest::global::LumiIntAnalyzer); DEFINE_FWK_MODULE(edmtest::global::RunSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::global::LumiSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::global::ProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockAnalyzerThreeTags); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockAnalyzerReuseCache); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockIntAnalyzerNoRegistration); diff --git a/FWCore/Framework/test/stubs/TestGlobalFilters.cc b/FWCore/Framework/test/stubs/TestGlobalFilters.cc index 89e480b02690b..5845b4ad7c5eb 100644 --- a/FWCore/Framework/test/stubs/TestGlobalFilters.cc +++ b/FWCore/Framework/test/stubs/TestGlobalFilters.cc @@ -6,24 +6,25 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include #include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/global/EDFilter.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/global/EDFilter.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace global { @@ -436,7 +437,7 @@ namespace edmtest { } } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "ProcessBlockIntFilter::begin transitions " << m_count << " but it was supposed to be " << 0; @@ -459,7 +460,7 @@ namespace edmtest { return true; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -512,7 +513,7 @@ namespace edmtest { } } - void beginProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void beginProcessBlockProduce(edm::ProcessBlock& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "TestBeginProcessBlockFilter transitions " << m_count << " but it was supposed to be " << 0; @@ -569,7 +570,7 @@ namespace edmtest { return true; } - void endProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void endProcessBlockProduce(edm::ProcessBlock& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -753,6 +754,141 @@ namespace edmtest { } }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntFilter + : public edm::global::EDFilter< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntFilter(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + bool filter(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + return true; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntFilter transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace global } // namespace edmtest @@ -768,3 +904,4 @@ DEFINE_FWK_MODULE(edmtest::global::TestBeginRunFilter); DEFINE_FWK_MODULE(edmtest::global::TestBeginLumiBlockFilter); DEFINE_FWK_MODULE(edmtest::global::TestEndRunFilter); DEFINE_FWK_MODULE(edmtest::global::TestEndLumiBlockFilter); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockIntFilter); diff --git a/FWCore/Framework/test/stubs/TestGlobalProducers.cc b/FWCore/Framework/test/stubs/TestGlobalProducers.cc index 03fd019d42b83..0b35ea19f8dd5 100644 --- a/FWCore/Framework/test/stubs/TestGlobalProducers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalProducers.cc @@ -6,25 +6,28 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/global/EDProducer.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/global/EDProducer.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" -#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace global { @@ -481,7 +484,7 @@ namespace edmtest { } } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "ProcessBlockIntProducer::begin transitions " << m_count << " but it was supposed to be " << 0; @@ -504,7 +507,7 @@ namespace edmtest { ++m_count; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -557,7 +560,7 @@ namespace edmtest { } } - void beginProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void beginProcessBlockProduce(edm::ProcessBlock& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "TestBeginProcessBlockProducer transitions " << m_count << " but it was supposed to be " << 0; @@ -610,7 +613,7 @@ namespace edmtest { void produce(edm::StreamID iID, edm::Event&, edm::EventSetup const&) const override { ++m_count; } - void endProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void endProcessBlockProduce(edm::ProcessBlock& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -809,6 +812,141 @@ namespace edmtest { const unsigned int m_expectedCount; const edm::EDPutTokenT m_putToken; }; + + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntProducer + : public edm::global::EDProducer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntProducer(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + void produce(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntProducer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace global } // namespace edmtest @@ -826,3 +964,4 @@ DEFINE_FWK_MODULE(edmtest::global::TestEndRunProducer); DEFINE_FWK_MODULE(edmtest::global::TestBeginLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::global::TestEndLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::global::TestAccumulator); +DEFINE_FWK_MODULE(edmtest::global::InputProcessBlockIntProducer); diff --git a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc index b702c93acc500..eef2fac9615c1 100644 --- a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc @@ -6,25 +6,28 @@ edm::*Cache templates for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/limited/EDAnalyzer.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/limited/EDAnalyzer.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" -#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace limited { @@ -342,7 +345,7 @@ namespace edmtest { } } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "ProcessBlockIntAnalyzer::begin transitions " << m_count << " but it was supposed to be " << 0; @@ -364,7 +367,7 @@ namespace edmtest { ++m_count; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -404,6 +407,143 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntAnalyzer + : public edm::limited::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntAnalyzer(edm::ParameterSet const& pset) + : edm::limited::EDAnalyzerBase(pset), + edm::limited::EDAnalyzer< + edm::InputProcessBlockCache>(pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntAnalyzer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace limited } // namespace edmtest @@ -413,3 +553,4 @@ DEFINE_FWK_MODULE(edmtest::limited::LumiIntAnalyzer); DEFINE_FWK_MODULE(edmtest::limited::RunSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::limited::LumiSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::limited::ProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::limited::InputProcessBlockIntAnalyzer); diff --git a/FWCore/Framework/test/stubs/TestLimitedFilters.cc b/FWCore/Framework/test/stubs/TestLimitedFilters.cc index e0ec00d2e0f39..4af2499a561fa 100644 --- a/FWCore/Framework/test/stubs/TestLimitedFilters.cc +++ b/FWCore/Framework/test/stubs/TestLimitedFilters.cc @@ -6,24 +6,26 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/limited/EDFilter.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/limited/EDFilter.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace limited { @@ -454,7 +456,7 @@ namespace edmtest { } } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "ProcessBlockIntFilter::begin transitions " << m_count << " but it was supposed to be " << 0; @@ -477,7 +479,7 @@ namespace edmtest { return true; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -531,7 +533,7 @@ namespace edmtest { } } - void beginProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void beginProcessBlockProduce(edm::ProcessBlock& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "TestBeginProcessBlockFilter transitions " << m_count << " but it was supposed to be " << 0; @@ -589,7 +591,7 @@ namespace edmtest { return true; } - void endProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void endProcessBlockProduce(edm::ProcessBlock& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -785,6 +787,144 @@ namespace edmtest { } }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntFilter + : public edm::limited::EDFilter< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntFilter(edm::ParameterSet const& pset) + : edm::limited::EDFilterBase(pset), + edm::limited::EDFilter< + edm::InputProcessBlockCache>(pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + bool filter(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + return true; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntFilter transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace limited } // namespace edmtest @@ -800,3 +940,4 @@ DEFINE_FWK_MODULE(edmtest::limited::TestBeginRunFilter); DEFINE_FWK_MODULE(edmtest::limited::TestBeginLumiBlockFilter); DEFINE_FWK_MODULE(edmtest::limited::TestEndRunFilter); DEFINE_FWK_MODULE(edmtest::limited::TestEndLumiBlockFilter); +DEFINE_FWK_MODULE(edmtest::limited::InputProcessBlockIntFilter); diff --git a/FWCore/Framework/test/stubs/TestLimitedProducers.cc b/FWCore/Framework/test/stubs/TestLimitedProducers.cc index 28d968715b86e..300a816255c6e 100644 --- a/FWCore/Framework/test/stubs/TestLimitedProducers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedProducers.cc @@ -6,24 +6,26 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/limited/EDProducer.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/limited/EDProducer.h" +#include "FWCore/Framework/interface/ProcessBlock.h" +#include "FWCore/Framework/interface/Run.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace limited { @@ -501,7 +503,7 @@ namespace edmtest { } } - void beginProcessBlock(edm::ProcessBlock const& processBlock) const override { + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "ProcessBlockIntProducer::begin transitions " << m_count << " but it was supposed to be " << 0; @@ -523,7 +525,7 @@ namespace edmtest { ++m_count; } - void endProcessBlock(edm::ProcessBlock const& processBlock) const override { + void endProcessBlock(edm::ProcessBlock const& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -578,7 +580,7 @@ namespace edmtest { } } - void beginProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void beginProcessBlockProduce(edm::ProcessBlock& processBlock) override { if (m_count != 0) { throw cms::Exception("transitions") << "TestBeginProcessBlockProducer transitions " << m_count << " but it was supposed to be " << 0; @@ -634,7 +636,7 @@ namespace edmtest { void produce(edm::StreamID iID, edm::Event&, edm::EventSetup const&) const override { ++m_count; } - void endProcessBlockProduce(edm::ProcessBlock& processBlock) const override { + void endProcessBlockProduce(edm::ProcessBlock& processBlock) override { ++m_count; if (m_count != trans_) { throw cms::Exception("transitions") @@ -848,6 +850,144 @@ namespace edmtest { const unsigned int m_expectedCount; const edm::EDPutTokenT m_putToken; }; + + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntProducer + : public edm::limited::EDProducer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntProducer(edm::ParameterSet const& pset) + : edm::limited::EDProducerBase(pset), + edm::limited::EDProducer< + edm::InputProcessBlockCache>(pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + void produce(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntProducer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace limited } // namespace edmtest @@ -865,3 +1005,4 @@ DEFINE_FWK_MODULE(edmtest::limited::TestEndRunProducer); DEFINE_FWK_MODULE(edmtest::limited::TestBeginLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::limited::TestEndLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::limited::TestAccumulator); +DEFINE_FWK_MODULE(edmtest::limited::InputProcessBlockIntProducer); diff --git a/FWCore/Framework/test/stubs/TestModuleDelete.cc b/FWCore/Framework/test/stubs/TestModuleDelete.cc index 524a75b24e9e6..5475582403f1d 100644 --- a/FWCore/Framework/test/stubs/TestModuleDelete.cc +++ b/FWCore/Framework/test/stubs/TestModuleDelete.cc @@ -166,7 +166,7 @@ namespace edmtest { desc.addUntracked>("srcEvent", std::vector{}); descriptions.addDefault(desc); } - void beginProcessBlockProduce(edm::ProcessBlock&) const override { + void beginProcessBlockProduce(edm::ProcessBlock&) override { throw edm::Exception(edm::errors::NotFound) << "Intentional 'NotFound' exception for testing purposes\n"; } void produce(edm::StreamID, edm::Event& e, edm::EventSetup const& c) const override { diff --git a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc index 29674a6749e4d..fb5b70c650d47 100644 --- a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc @@ -6,25 +6,29 @@ edm::one cache templates for testing purposes only. ----------------------------------------------------------------------*/ -#include #include +#include +#include +#include #include -#include -#include + +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" #include "FWCore/Framework/interface/one/EDAnalyzer.h" #include "FWCore/Framework/interface/ProcessBlock.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" #include "FWCore/Framework/interface/Run.h" -#include "FWCore/Framework/interface/LuminosityBlock.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" -#include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" -#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace one { @@ -312,6 +316,261 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntAnalyzer + : public edm::one::EDAnalyzer< + edm::WatchProcessBlock, + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntAnalyzer(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + expectedFillerSum_ = pset.getUntrackedParameter("expectedFillerSum", 0); + expectedCacheSize_ = pset.getUntrackedParameter("expectedCacheSize", 0); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockNotFound"); + if (not tag.label().empty()) { + getTokenBeginNotFound_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockNotFound"); + if (not tag.label().empty()) { + getTokenEndNotFound_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesProcessBlockNotFound1"); + if (not tag.label().empty()) { + getTokenNotFound1_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesProcessBlockNotFound2"); + if (not tag.label().empty()) { + getTokenNotFound2_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesProcessBlockNotFound3"); + if (not tag.label().empty()) { + getTokenNotFound3_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesProcessBlockNotFound4"); + if (not tag.label().empty()) { + getTokenNotFound4_ = consumes(tag); + } + } + + if (!getTokenBegin_.isUninitialized()) { + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + fillerSum_ += processBlock.get(getTokenBegin_).value; + fillerSum_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + if (!getTokenBegin_.isUninitialized()) { + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + fillerSum_ += processBlock.get(getTokenBegin_).value; + fillerSum_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + if (!getTokenBegin_.isUninitialized()) { + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + fillerSum_ += processBlock.get(getTokenBegin_).value; + fillerSum_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + } + + void beginProcessBlock(edm::ProcessBlock const& processBlock) override { + if (!getTokenBeginNotFound_.isUninitialized() && processBlock.getHandle(getTokenBeginNotFound_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (begin)"; + } + } + + void endProcessBlock(edm::ProcessBlock const& processBlock) override { + if (!getTokenEndNotFound_.isUninitialized() && processBlock.getHandle(getTokenEndNotFound_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (end)"; + } + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + if (!getTokenBegin_.isUninitialized() && processBlock.getHandle(getTokenBegin_).isValid()) { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + } + if (processBlock.processName() == "MERGE") { + if (!getTokenBeginM_.isUninitialized() && processBlock.getHandle(getTokenBeginM_).isValid()) { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + } + + if (!getTokenNotFound1_.isUninitialized() && processBlock.getHandle(getTokenNotFound1_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (token 1)"; + } + if (!getTokenNotFound2_.isUninitialized() && processBlock.getHandle(getTokenNotFound2_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (token 2)"; + } + if (!getTokenNotFound3_.isUninitialized() && processBlock.getHandle(getTokenNotFound3_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (token 3)"; + } + if (!getTokenNotFound4_.isUninitialized() && processBlock.getHandle(getTokenNotFound4_).isValid()) { + throw cms::Exception("TestFailure") << "Expected handle to be invalid but it is valid (token 4)"; + } + + ++transitions_; + } + + void analyze(edm::Event const& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + + if (expectedCacheSize_ != 0u && expectedCacheSize_ != cacheSize()) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze, unexpected cacheSize " + << cacheSize() << " but it was supposed to be " << expectedCacheSize_; + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntAnalyzer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (expectedFillerSum_ != 0 && fillerSum_ != expectedFillerSum_) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer fillerSum " << fillerSum_ + << " but it was supposed to be " << expectedFillerSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer cache size not zero at endJob " << cacheSize(); + } + } + + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.add("transitions"); + desc.add>("expectedByRun", std::vector()); + desc.add("expectedSum"); + edm::InputTag defaultInputTag; + desc.addUntracked("expectedFillerSum", 0); + desc.addUntracked("expectedCacheSize", 0); + desc.add("consumesBeginProcessBlock", defaultInputTag); + desc.add("consumesEndProcessBlock", defaultInputTag); + desc.add("consumesBeginProcessBlockM", defaultInputTag); + desc.add("consumesEndProcessBlockM", defaultInputTag); + desc.add("consumesBeginProcessBlockNotFound", defaultInputTag); + desc.add("consumesEndProcessBlockNotFound", defaultInputTag); + desc.add("consumesProcessBlockNotFound1", defaultInputTag); + desc.add("consumesProcessBlockNotFound2", defaultInputTag); + desc.add("consumesProcessBlockNotFound3", defaultInputTag); + desc.add("consumesProcessBlockNotFound4", defaultInputTag); + descriptions.addDefault(desc); + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + edm::EDGetTokenT getTokenBeginNotFound_; + edm::EDGetTokenT getTokenEndNotFound_; + edm::EDGetTokenT getTokenNotFound1_; + edm::EDGetTokenT getTokenNotFound2_; + edm::EDGetTokenT getTokenNotFound3_; + edm::EDGetTokenT getTokenNotFound4_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + int fillerSum_{0}; + int expectedFillerSum_{0}; + unsigned int expectedCacheSize_{0}; + }; + } // namespace one } // namespace edmtest @@ -321,3 +580,4 @@ DEFINE_FWK_MODULE(edmtest::one::WatchLumiBlocksAnalyzer); DEFINE_FWK_MODULE(edmtest::one::RunCacheAnalyzer); DEFINE_FWK_MODULE(edmtest::one::LumiBlockCacheAnalyzer); DEFINE_FWK_MODULE(edmtest::one::ProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::one::InputProcessBlockIntAnalyzer); diff --git a/FWCore/Framework/test/stubs/TestOneFilters.cc b/FWCore/Framework/test/stubs/TestOneFilters.cc index ffa99ecf6bd90..b684185cc36e6 100644 --- a/FWCore/Framework/test/stubs/TestOneFilters.cc +++ b/FWCore/Framework/test/stubs/TestOneFilters.cc @@ -6,24 +6,26 @@ edm::one cache classes and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include #include -#include -#include + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" #include "FWCore/Framework/interface/one/EDFilter.h" #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/Framework/interface/Run.h" -#include "FWCore/Framework/interface/LuminosityBlock.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" -#include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace one { @@ -550,6 +552,141 @@ namespace edmtest { } }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntFilter + : public edm::one::EDFilter< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntFilter(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + bool filter(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + return true; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntFilter transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace one } // namespace edmtest @@ -565,3 +702,4 @@ DEFINE_FWK_MODULE(edmtest::one::BeginRunFilter); DEFINE_FWK_MODULE(edmtest::one::BeginLumiBlockFilter); DEFINE_FWK_MODULE(edmtest::one::EndRunFilter); DEFINE_FWK_MODULE(edmtest::one::EndLumiBlockFilter); +DEFINE_FWK_MODULE(edmtest::one::InputProcessBlockIntFilter); diff --git a/FWCore/Framework/test/stubs/TestOneProducers.cc b/FWCore/Framework/test/stubs/TestOneProducers.cc index eef2ab1bf3f35..a79142f254bb7 100644 --- a/FWCore/Framework/test/stubs/TestOneProducers.cc +++ b/FWCore/Framework/test/stubs/TestOneProducers.cc @@ -6,24 +6,26 @@ edm::one cache classes and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include +#include +#include #include -#include -#include -#include "FWCore/Framework/interface/one/EDProducer.h" -#include "FWCore/Framework/src/WorkerT.h" -#include "FWCore/Framework/interface/HistoryAppender.h" -#include "FWCore/ServiceRegistry/interface/ParentContext.h" -#include "FWCore/ServiceRegistry/interface/StreamContext.h" -#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/one/EDProducer.h" #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/Framework/interface/Run.h" -#include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/EDGetToken.h" +#include "FWCore/Utilities/interface/InputTag.h" namespace edmtest { namespace one { @@ -574,6 +576,140 @@ namespace edmtest { const edm::EDPutTokenT m_putToken; }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntProducer + : public edm::one::EDProducer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntProducer(edm::ParameterSet const& pset) { + expectedTransitions_ = pset.getParameter("transitions"); + expectedByRun_ = pset.getParameter>("expectedByRun"); + expectedSum_ = pset.getParameter("expectedSum"); + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++transitions_; + return returnValue; + }); + } + + void accessInputProcessBlock(edm::ProcessBlock const& processBlock) override { + if (processBlock.processName() == "PROD1") { + sum_ += processBlock.get(getTokenBegin_).value; + sum_ += processBlock.get(getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + sum_ += processBlock.get(getTokenBeginM_).value; + sum_ += processBlock.get(getTokenEndM_).value; + } + ++transitions_; + } + + void produce(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + ++transitions_; + } + + void endJob() override { + if (transitions_ != expectedTransitions_) { + throw cms::Exception("transitions") << "InputProcessBlockIntProducer transitions " << transitions_ + << " but it was supposed to be " << expectedTransitions_; + } + if (sum_ != expectedSum_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer sum " << sum_ << " but it was supposed to be " << expectedSum_; + } + if (cacheSize() > 0u) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer cache size not zero at endJob " << cacheSize(); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + }; + } // namespace one } // namespace edmtest @@ -590,3 +726,4 @@ DEFINE_FWK_MODULE(edmtest::one::TestBeginLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::one::TestEndRunProducer); DEFINE_FWK_MODULE(edmtest::one::TestEndLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::one::TestAccumulator); +DEFINE_FWK_MODULE(edmtest::one::InputProcessBlockIntProducer); diff --git a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc index e13523b8bdbfd..acf31360c1b19 100644 --- a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc @@ -6,11 +6,15 @@ edm::*Cache templates for testing purposes only. ----------------------------------------------------------------------*/ -#include #include -#include -#include #include +#include +#include +#include +#include +#include + +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/stream/EDAnalyzer.h" #include "FWCore/Framework/src/WorkerT.h" #include "FWCore/Framework/interface/HistoryAppender.h" @@ -21,10 +25,12 @@ for testing purposes only. #include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/Framework/interface/TriggerNamesService.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include "FWCore/Utilities/interface/EDMException.h" #include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" namespace edmtest { namespace stream { @@ -497,10 +503,6 @@ namespace edmtest { } } - static std::shared_ptr accessInputProcessBlock(edm::ProcessBlock const&, TestGlobalCacheAn*) { - return std::make_shared(); - } - void analyze(edm::Event const&, edm::EventSetup const&) override { TestGlobalCacheAn const* testGlobalCache = globalCache(); if (testGlobalCache->m_count < 1u) { @@ -552,6 +554,263 @@ namespace edmtest { } }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntAnalyzer + : public edm::stream::EDAnalyzer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntAnalyzer(edm::ParameterSet const& pset) { + { + expectedByRun_ = pset.getParameter>("expectedByRun"); + sleepTime_ = pset.getParameter("sleepTime"); + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + return returnValue; + }); + } + + static void accessInputProcessBlock(edm::ProcessBlock const&) { + edm::LogAbsolute("InputProcessBlockIntAnalyzer") << "InputProcessBlockIntAnalyzer::accessInputProcessBlock"; + } + + void analyze(edm::Event const& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + // Force events to be processed concurrently + if (sleepTime_ > 0) { + usleep(sleepTime_); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + std::vector expectedByRun_; + unsigned int sleepTime_{0}; + }; + + struct InputProcessBlockGlobalCacheAn { + // The tokens are duplicated in this test module to prove that they + // work both as GlobalCache members and module data members. + // We need them as GlobalCache members for accessInputProcessBlock. + // In registerProcessBlockCacheFiller we use tokens that are member + // variables of the class and because the lambda captures the "this" + // pointer of the zeroth stream module instance. We always + // use the zeroth EDConsumer. In the case of registerProcessBlockCacheFiller, + // either set of tokens would work. Note that in the GlobalCache case + // there is a slight weirdness that the zeroth consumer is used but + // the token comes from the last consumer instance. It works because + // all the stream module instances have EDConsumer base classes with + // containers with the same contents in the same order (not 100% guaranteed, + // but it would be difficult to implement a module where this isn't true). + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + unsigned int sleepTime_{0}; + }; + + // Same thing as previous class except with a GlobalCache added + class InputProcessBlockIntAnalyzerG + : public edm::stream::EDAnalyzer< + edm::InputProcessBlockCache, + edm::GlobalCache> { + public: + explicit InputProcessBlockIntAnalyzerG(edm::ParameterSet const& pset, + InputProcessBlockGlobalCacheAn const* testGlobalCache) { + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + testGlobalCache->getTokenBegin_ = getTokenBegin_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + testGlobalCache->getTokenEnd_ = getTokenEnd_; + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + testGlobalCache->getTokenBeginM_ = getTokenBeginM_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + testGlobalCache->getTokenEndM_ = getTokenEndM_; + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + } + + static std::unique_ptr initializeGlobalCache(edm::ParameterSet const& pset) { + auto testGlobalCache = std::make_unique(); + testGlobalCache->expectedTransitions_ = pset.getParameter("transitions"); + testGlobalCache->expectedByRun_ = pset.getParameter>("expectedByRun"); + testGlobalCache->expectedSum_ = pset.getParameter("expectedSum"); + testGlobalCache->sleepTime_ = pset.getParameter("sleepTime"); + return testGlobalCache; + } + + static void accessInputProcessBlock(edm::ProcessBlock const& processBlock, + InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (processBlock.processName() == "PROD1") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBegin_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBeginM_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEndM_).value; + } + ++testGlobalCache->transitions_; + } + + void analyze(edm::Event const& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + auto testGlobalCache = globalCache(); + if (!testGlobalCache->expectedByRun_.empty()) { + if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzerG::analyze cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzerG::analyze second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzerG::analyze third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + } + ++testGlobalCache->transitions_; + + // Force events to be processed concurrently + if (testGlobalCache->sleepTime_ > 0) { + usleep(testGlobalCache->sleepTime_); + } + } + + static void globalEndJob(InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (testGlobalCache->transitions_ != testGlobalCache->expectedTransitions_) { + throw cms::Exception("transitions") + << "InputProcessBlockIntAnalyzerG transitions " << testGlobalCache->transitions_ + << " but it was supposed to be " << testGlobalCache->expectedTransitions_; + } + + if (testGlobalCache->sum_ != testGlobalCache->expectedSum_) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzerG sum " << testGlobalCache->sum_ + << " but it was supposed to be " << testGlobalCache->expectedSum_; + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + }; + + // The next two test that modules without the + // static accessInputProcessBlock function will build. + // And also that modules with no functor registered run. + + class InputProcessBlockIntAnalyzerNS + : public edm::stream::EDAnalyzer> { + public: + explicit InputProcessBlockIntAnalyzerNS(edm::ParameterSet const& pset) {} + void analyze(edm::Event const&, edm::EventSetup const&) override {} + }; + + // Same thing as previous class except with a GlobalCache added + class InputProcessBlockIntAnalyzerGNS + : public edm::stream::EDAnalyzer, + edm::GlobalCache> { + public: + explicit InputProcessBlockIntAnalyzerGNS(edm::ParameterSet const& pset, + TestGlobalCacheAn const* testGlobalCache) {} + static std::unique_ptr initializeGlobalCache(edm::ParameterSet const&) { + return std::make_unique(); + } + void analyze(edm::Event const&, edm::EventSetup const&) override {} + static void globalEndJob(TestGlobalCacheAn* testGlobalCache) {} + }; + } // namespace stream } // namespace edmtest std::atomic edmtest::stream::GlobalIntAnalyzer::m_count{0}; @@ -593,3 +852,7 @@ DEFINE_FWK_MODULE(edmtest::stream::LumiIntAnalyzer); DEFINE_FWK_MODULE(edmtest::stream::RunSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::stream::LumiSummaryIntAnalyzer); DEFINE_FWK_MODULE(edmtest::stream::ProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntAnalyzer); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntAnalyzerG); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntAnalyzerNS); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntAnalyzerGNS); diff --git a/FWCore/Framework/test/stubs/TestStreamFilters.cc b/FWCore/Framework/test/stubs/TestStreamFilters.cc index 1511cd850919e..d8a33f1b84a28 100644 --- a/FWCore/Framework/test/stubs/TestStreamFilters.cc +++ b/FWCore/Framework/test/stubs/TestStreamFilters.cc @@ -6,14 +6,20 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include -#include -#include #include +#include +#include +#include +#include +#include + +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/stream/EDFilter.h" #include "FWCore/Framework/src/WorkerT.h" #include "FWCore/Framework/interface/HistoryAppender.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ServiceRegistry/interface/ParentContext.h" #include "FWCore/ServiceRegistry/interface/StreamContext.h" #include "FWCore/Utilities/interface/GlobalIdentifier.h" @@ -22,6 +28,7 @@ for testing purposes only. #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" namespace edmtest { namespace stream { @@ -478,10 +485,6 @@ namespace edmtest { } } - static std::shared_ptr accessInputProcessBlock(edm::ProcessBlock const&, TestGlobalCacheFil*) { - return std::make_shared(); - } - bool filter(edm::Event&, edm::EventSetup const&) override { TestGlobalCacheFil const* testGlobalCache = globalCache(); if (testGlobalCache->m_count < 1u) { @@ -889,6 +892,240 @@ namespace edmtest { } }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntFilter + : public edm::stream::EDFilter< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntFilter(edm::ParameterSet const& pset) { + { + expectedByRun_ = pset.getParameter>("expectedByRun"); + sleepTime_ = pset.getParameter("sleepTime"); + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + return returnValue; + }); + } + + static void accessInputProcessBlock(edm::ProcessBlock const&) { + edm::LogAbsolute("InputProcessBlockIntFilter") << "InputProcessBlockIntFilter::accessInputProcessBlock"; + } + + bool filter(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + // Force events to be processed concurrently + if (sleepTime_ > 0) { + usleep(sleepTime_); + } + return true; + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + std::vector expectedByRun_; + unsigned int sleepTime_{0}; + }; + + struct InputProcessBlockGlobalCacheAn { + // The tokens are duplicated in this test module to prove that they + // work both as GlobalCache members and module data members. + // We need them as GlobalCache members for accessInputProcessBlock. + // In registerProcessBlockCacheFiller we use tokens that are member + // variables of the class and because the lambda captures the "this" + // pointer of the zeroth stream module instance. We always + // use the zeroth EDConsumer. In the case of registerProcessBlockCacheFiller, + // either set of tokens would work. Note that in the GlobalCache case + // there is a slight weirdness that the zeroth consumer is used but + // the token comes from the last consumer instance. It works because + // all the stream module instances have EDConsumer base classes with + // containers with the same contents in the same order (not 100% guaranteed, + // but it would be difficult to implement a module where this isn't true). + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + unsigned int sleepTime_{0}; + }; + + // Same thing as previous class except with a GlobalCache added + class InputProcessBlockIntFilterG + : public edm::stream::EDFilter< + edm::InputProcessBlockCache, + edm::GlobalCache> { + public: + explicit InputProcessBlockIntFilterG(edm::ParameterSet const& pset, + InputProcessBlockGlobalCacheAn const* testGlobalCache) { + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + testGlobalCache->getTokenBegin_ = getTokenBegin_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + testGlobalCache->getTokenEnd_ = getTokenEnd_; + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + testGlobalCache->getTokenBeginM_ = getTokenBeginM_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + testGlobalCache->getTokenEndM_ = getTokenEndM_; + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + } + + static std::unique_ptr initializeGlobalCache(edm::ParameterSet const& pset) { + auto testGlobalCache = std::make_unique(); + testGlobalCache->expectedTransitions_ = pset.getParameter("transitions"); + testGlobalCache->expectedByRun_ = pset.getParameter>("expectedByRun"); + testGlobalCache->expectedSum_ = pset.getParameter("expectedSum"); + testGlobalCache->sleepTime_ = pset.getParameter("sleepTime"); + return testGlobalCache; + } + + static void accessInputProcessBlock(edm::ProcessBlock const& processBlock, + InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (processBlock.processName() == "PROD1") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBegin_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBeginM_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEndM_).value; + } + ++testGlobalCache->transitions_; + } + + bool filter(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + auto testGlobalCache = globalCache(); + if (!testGlobalCache->expectedByRun_.empty()) { + if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilterG::filter cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilterG::filter second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilterG::filter third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + } + ++testGlobalCache->transitions_; + + // Force events to be processed concurrently + if (testGlobalCache->sleepTime_ > 0) { + usleep(testGlobalCache->sleepTime_); + } + return true; + } + + static void globalEndJob(InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (testGlobalCache->transitions_ != testGlobalCache->expectedTransitions_) { + throw cms::Exception("transitions") + << "InputProcessBlockIntFilterG transitions " << testGlobalCache->transitions_ + << " but it was supposed to be " << testGlobalCache->expectedTransitions_; + } + + if (testGlobalCache->sum_ != testGlobalCache->expectedSum_) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilterG sum " << testGlobalCache->sum_ + << " but it was supposed to be " << testGlobalCache->expectedSum_; + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + }; + } // namespace stream } // namespace edmtest std::atomic edmtest::stream::GlobalIntFilter::m_count{0}; @@ -952,3 +1189,5 @@ DEFINE_FWK_MODULE(edmtest::stream::TestBeginRunFilter); DEFINE_FWK_MODULE(edmtest::stream::TestEndRunFilter); DEFINE_FWK_MODULE(edmtest::stream::TestBeginLumiBlockFilter); DEFINE_FWK_MODULE(edmtest::stream::TestEndLumiBlockFilter); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntFilter); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntFilterG); diff --git a/FWCore/Framework/test/stubs/TestStreamProducers.cc b/FWCore/Framework/test/stubs/TestStreamProducers.cc index 8b97fd5c3fb58..c5197f11893f1 100644 --- a/FWCore/Framework/test/stubs/TestStreamProducers.cc +++ b/FWCore/Framework/test/stubs/TestStreamProducers.cc @@ -6,14 +6,20 @@ edm::*Cache templates and edm::*Producer classes for testing purposes only. ----------------------------------------------------------------------*/ -#include + #include -#include -#include #include +#include +#include +#include +#include +#include + +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/stream/EDProducer.h" #include "FWCore/Framework/src/WorkerT.h" #include "FWCore/Framework/interface/HistoryAppender.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ServiceRegistry/interface/ParentContext.h" #include "FWCore/ServiceRegistry/interface/StreamContext.h" #include "FWCore/Utilities/interface/GlobalIdentifier.h" @@ -22,6 +28,7 @@ for testing purposes only. #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" namespace edmtest { namespace stream { @@ -960,6 +967,238 @@ namespace edmtest { ~TestAccumulator() {} }; + class TestInputProcessBlockCache { + public: + long long int value_ = 0; + }; + + class TestInputProcessBlockCache1 { + public: + long long int value_ = 0; + }; + + class InputProcessBlockIntProducer + : public edm::stream::EDProducer< + edm::InputProcessBlockCache> { + public: + explicit InputProcessBlockIntProducer(edm::ParameterSet const& pset) { + { + expectedByRun_ = pset.getParameter>("expectedByRun"); + sleepTime_ = pset.getParameter("sleepTime"); + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + return returnValue; + }); + } + + static void accessInputProcessBlock(edm::ProcessBlock const&) { + edm::LogAbsolute("InputProcessBlockIntProducer") << "InputProcessBlockIntProducer::accessInputProcessBlock"; + } + + void produce(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + if (!expectedByRun_.empty()) { + if (expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << expectedByRun_[event.run()]; + } + } + // Force events to be processed concurrently + if (sleepTime_ > 0) { + usleep(sleepTime_); + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + std::vector expectedByRun_; + unsigned int sleepTime_{0}; + }; + + struct InputProcessBlockGlobalCacheAn { + // The tokens are duplicated in this test module to prove that they + // work both as GlobalCache members and module data members. + // We need them as GlobalCache members for accessInputProcessBlock. + // In registerProcessBlockCacheFiller we use tokens that are member + // variables of the class and because the lambda captures the "this" + // pointer of the zeroth stream module instance. We always + // use the zeroth EDConsumer. In the case of registerProcessBlockCacheFiller, + // either set of tokens would work. Note that in the GlobalCache case + // there is a slight weirdness that the zeroth consumer is used but + // the token comes from the last consumer instance. It works because + // all the stream module instances have EDConsumer base classes with + // containers with the same contents in the same order (not 100% guaranteed, + // but it would be difficult to implement a module where this isn't true). + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; + CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; + CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + int sum_{0}; + unsigned int expectedTransitions_{0}; + std::vector expectedByRun_; + int expectedSum_{0}; + unsigned int sleepTime_{0}; + }; + + // Same thing as previous class except with a GlobalCache added + class InputProcessBlockIntProducerG + : public edm::stream::EDProducer< + edm::InputProcessBlockCache, + edm::GlobalCache> { + public: + explicit InputProcessBlockIntProducerG(edm::ParameterSet const& pset, + InputProcessBlockGlobalCacheAn const* testGlobalCache) { + { + auto tag = pset.getParameter("consumesBeginProcessBlock"); + if (not tag.label().empty()) { + getTokenBegin_ = consumes(tag); + testGlobalCache->getTokenBegin_ = getTokenBegin_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlock"); + if (not tag.label().empty()) { + getTokenEnd_ = consumes(tag); + testGlobalCache->getTokenEnd_ = getTokenEnd_; + } + } + { + auto tag = pset.getParameter("consumesBeginProcessBlockM"); + if (not tag.label().empty()) { + getTokenBeginM_ = consumes(tag); + testGlobalCache->getTokenBeginM_ = getTokenBeginM_; + } + } + { + auto tag = pset.getParameter("consumesEndProcessBlockM"); + if (not tag.label().empty()) { + getTokenEndM_ = consumes(tag); + testGlobalCache->getTokenEndM_ = getTokenEndM_; + } + } + registerProcessBlockCacheFiller( + getTokenBegin_, [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + *returnValue += processBlock.get(getTokenBegin_).value; + *returnValue += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller<1>(getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + registerProcessBlockCacheFiller( + getTokenBegin_, + [this](edm::ProcessBlock const& processBlock, + std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(); + returnValue->value_ += processBlock.get(getTokenBegin_).value; + returnValue->value_ += processBlock.get(getTokenEnd_).value; + ++globalCache()->transitions_; + return returnValue; + }); + } + + static std::unique_ptr initializeGlobalCache(edm::ParameterSet const& pset) { + auto testGlobalCache = std::make_unique(); + testGlobalCache->expectedTransitions_ = pset.getParameter("transitions"); + testGlobalCache->expectedByRun_ = pset.getParameter>("expectedByRun"); + testGlobalCache->expectedSum_ = pset.getParameter("expectedSum"); + testGlobalCache->sleepTime_ = pset.getParameter("sleepTime"); + return testGlobalCache; + } + + static void accessInputProcessBlock(edm::ProcessBlock const& processBlock, + InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (processBlock.processName() == "PROD1") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBegin_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEnd_).value; + } + if (processBlock.processName() == "MERGE") { + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenBeginM_).value; + testGlobalCache->sum_ += processBlock.get(testGlobalCache->getTokenEndM_).value; + } + ++testGlobalCache->transitions_; + } + + void produce(edm::Event& event, edm::EventSetup const&) override { + auto cacheTuple = processBlockCaches(event); + auto testGlobalCache = globalCache(); + if (!testGlobalCache->expectedByRun_.empty()) { + if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducerG::produce cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducerG::produce second cached value was " << std::get<1>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + if (testGlobalCache->expectedByRun_[event.run()] != + std::get>(cacheTuple)->value_) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducerG::produce third cached value was " + << std::get>(cacheTuple)->value_ + << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + } + } + ++testGlobalCache->transitions_; + + // Force events to be processed concurrently + if (testGlobalCache->sleepTime_ > 0) { + usleep(testGlobalCache->sleepTime_); + } + } + + static void globalEndJob(InputProcessBlockGlobalCacheAn* testGlobalCache) { + if (testGlobalCache->transitions_ != testGlobalCache->expectedTransitions_) { + throw cms::Exception("transitions") + << "InputProcessBlockIntProducerG transitions " << testGlobalCache->transitions_ + << " but it was supposed to be " << testGlobalCache->expectedTransitions_; + } + + if (testGlobalCache->sum_ != testGlobalCache->expectedSum_) { + throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducerG sum " << testGlobalCache->sum_ + << " but it was supposed to be " << testGlobalCache->expectedSum_; + } + } + + private: + edm::EDGetTokenT getTokenBegin_; + edm::EDGetTokenT getTokenEnd_; + edm::EDGetTokenT getTokenBeginM_; + edm::EDGetTokenT getTokenEndM_; + }; + } // namespace stream } // namespace edmtest std::atomic edmtest::stream::GlobalIntProducer::m_count{0}; @@ -1031,3 +1270,5 @@ DEFINE_FWK_MODULE(edmtest::stream::TestEndRunProducer); DEFINE_FWK_MODULE(edmtest::stream::TestBeginLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::stream::TestEndLumiBlockProducer); DEFINE_FWK_MODULE(edmtest::stream::TestAccumulator); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntProducer); +DEFINE_FWK_MODULE(edmtest::stream::InputProcessBlockIntProducerG); diff --git a/FWCore/Framework/test/stubs/ToyIntProducers.cc b/FWCore/Framework/test/stubs/ToyIntProducers.cc index aa568acd426a7..db3ead0e8f3f6 100644 --- a/FWCore/Framework/test/stubs/ToyIntProducers.cc +++ b/FWCore/Framework/test/stubs/ToyIntProducers.cc @@ -517,7 +517,8 @@ namespace edmtest { edm::EndLuminosityBlockProducer, edm::EndRunProducer, edm::BeginProcessBlockProducer, - edm::EndProcessBlockProducer> { + edm::EndProcessBlockProducer, + edm::InputProcessBlockCache<>> { public: explicit NonEventIntProducer(edm::ParameterSet const& p) : bpbToken_{produces("beginProcessBlock")}, @@ -533,7 +534,8 @@ namespace edmtest { blExpect_{p.getUntrackedParameter("expectBeginLuminosityBlock")}, elExpect_{p.getUntrackedParameter("expectEndLuminosityBlock")}, erExpect_{p.getUntrackedParameter("expectEndRun")}, - epbExpect_{p.getUntrackedParameter("expectEndProcessBlock")} { + epbExpect_{p.getUntrackedParameter("expectEndProcessBlock")}, + aipbExpect_{p.getUntrackedParameter("expectAccessInputProcessBlock")} { { auto tag = p.getParameter("consumesBeginProcessBlock"); if (not tag.label().empty()) { @@ -570,14 +572,21 @@ namespace edmtest { epbGet_ = consumes(tag); } } + { + auto tag = p.getParameter("consumesAccessInputProcessBlock"); + if (not tag.label().empty()) { + aipbGet_ = consumes(tag); + } + } } void accumulate(edm::StreamID iID, edm::Event const& e, edm::EventSetup const& c) const override; - void beginProcessBlockProduce(edm::ProcessBlock&) const override; - void endProcessBlockProduce(edm::ProcessBlock&) const override; + void beginProcessBlockProduce(edm::ProcessBlock&) override; + void endProcessBlockProduce(edm::ProcessBlock&) override; void globalBeginRunProduce(edm::Run& e, edm::EventSetup const&) const override; void globalEndRunProduce(edm::Run& e, edm::EventSetup const&) const override; void globalBeginLuminosityBlockProduce(edm::LuminosityBlock& e, edm::EventSetup const&) const override; void globalEndLuminosityBlockProduce(edm::LuminosityBlock& e, edm::EventSetup const&) const override; + void accessInputProcessBlock(edm::ProcessBlock const&) override; static void fillDescriptions(edm::ConfigurationDescriptions& conf) { edm::ParameterSetDescription desc; @@ -587,6 +596,8 @@ namespace edmtest { desc.addUntracked("expectBeginProcessBlock", 0); desc.add("consumesEndProcessBlock", {}); desc.addUntracked("expectEndProcessBlock", 0); + desc.add("consumesAccessInputProcessBlock", {}); + desc.addUntracked("expectAccessInputProcessBlock", 0); desc.add("consumesBeginRun", {}); desc.addUntracked("expectBeginRun", 0); desc.add("consumesEndRun", {}); @@ -615,16 +626,18 @@ namespace edmtest { edm::EDGetTokenT elGet_; edm::EDGetTokenT erGet_; edm::EDGetTokenT epbGet_; + edm::EDGetTokenT aipbGet_; const int bpbExpect_; const int brExpect_; const int blExpect_; const int elExpect_; const int erExpect_; const int epbExpect_; + const int aipbExpect_; }; void NonEventIntProducer::accumulate(edm::StreamID iID, edm::Event const& e, edm::EventSetup const&) const {} - void NonEventIntProducer::beginProcessBlockProduce(edm::ProcessBlock& processBlock) const { + void NonEventIntProducer::beginProcessBlockProduce(edm::ProcessBlock& processBlock) { if (not bpbGet_.isUninitialized()) { check(processBlock.get(bpbGet_), bpbExpect_); } @@ -637,7 +650,7 @@ namespace edmtest { } processBlock.emplace(bpbToken_, value_); } - void NonEventIntProducer::endProcessBlockProduce(edm::ProcessBlock& processBlock) const { + void NonEventIntProducer::endProcessBlockProduce(edm::ProcessBlock& processBlock) { if (not epbGet_.isUninitialized()) { check(processBlock.get(epbGet_), epbExpect_); } @@ -646,6 +659,14 @@ namespace edmtest { } processBlock.emplace(epbToken_, value_); } + void NonEventIntProducer::accessInputProcessBlock(edm::ProcessBlock const& processBlock) { + if (not aipbGet_.isUninitialized()) { + check(processBlock.get(aipbGet_), aipbExpect_); + } + if (sleepTime_ > 0) { + usleep(sleepTime_); + } + } void NonEventIntProducer::globalBeginRunProduce(edm::Run& r, edm::EventSetup const&) const { if (not brGet_.isUninitialized()) { check(r.get(brGet_), brExpect_); @@ -697,14 +718,14 @@ namespace edmtest { explicit IntProducerBeginProcessBlock(edm::ParameterSet const& p) : token_{produces()}, value_(p.getParameter("ivalue")) {} void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} - void beginProcessBlockProduce(edm::ProcessBlock&) const override; + void beginProcessBlockProduce(edm::ProcessBlock&) override; private: edm::EDPutTokenT token_; int value_; }; - void IntProducerBeginProcessBlock::beginProcessBlockProduce(edm::ProcessBlock& processBlock) const { + void IntProducerBeginProcessBlock::beginProcessBlockProduce(edm::ProcessBlock& processBlock) { processBlock.emplace(token_, value_); } @@ -721,7 +742,7 @@ namespace edmtest { token4_{produces("four")}, value_(p.getParameter("ivalue")) {} void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} - void endProcessBlockProduce(edm::ProcessBlock&) const override; + void endProcessBlockProduce(edm::ProcessBlock&) override; private: edm::EDPutTokenT token_; @@ -731,13 +752,34 @@ namespace edmtest { int value_; }; - void IntProducerEndProcessBlock::endProcessBlockProduce(edm::ProcessBlock& processBlock) const { + void IntProducerEndProcessBlock::endProcessBlockProduce(edm::ProcessBlock& processBlock) { processBlock.emplace(token_, value_); processBlock.emplace(token2_, value_ + 2); processBlock.put(token3_, std::make_unique(value_ + 3)); processBlock.put(token4_, std::make_unique(value_ + 4)); } + //-------------------------------------------------------------------- + // + // Produces an TransientIntProduct in ProcessBlock at endProcessBlock + // + class TransientIntProducerEndProcessBlock : public edm::global::EDProducer { + public: + explicit TransientIntProducerEndProcessBlock(edm::ParameterSet const& p) + : token_{produces()}, + value_(p.getParameter("ivalue")) {} + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + void endProcessBlockProduce(edm::ProcessBlock&) override; + + private: + edm::EDPutTokenT token_; + int value_; + }; + + void TransientIntProducerEndProcessBlock::endProcessBlockProduce(edm::ProcessBlock& processBlock) { + processBlock.emplace(token_, value_); + } + //-------------------------------------------------------------------- // // Produces an IntProduct instance, the module must get run, otherwise an exception is thrown @@ -807,6 +849,7 @@ using edmtest::ManyIntWhenRegisteredProducer; using edmtest::NonEventIntProducer; using edmtest::NonProducer; using edmtest::TransientIntProducer; +using edmtest::TransientIntProducerEndProcessBlock; DEFINE_FWK_MODULE(FailingProducer); DEFINE_FWK_MODULE(edmtest::FailingInLumiProducer); DEFINE_FWK_MODULE(edmtest::FailingInRunProducer); @@ -827,4 +870,5 @@ DEFINE_FWK_MODULE(ManyIntWhenRegisteredProducer); DEFINE_FWK_MODULE(NonEventIntProducer); DEFINE_FWK_MODULE(IntProducerBeginProcessBlock); DEFINE_FWK_MODULE(IntProducerEndProcessBlock); +DEFINE_FWK_MODULE(TransientIntProducerEndProcessBlock); DEFINE_FWK_MODULE(edmtest::MustRunIntProducer); diff --git a/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc b/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc new file mode 100644 index 0000000000000..13894c38add9d --- /dev/null +++ b/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc @@ -0,0 +1,68 @@ +#include "catch.hpp" + +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" +#include "FWCore/Framework/interface/moduleAbilities.h" +#include "FWCore/Framework/interface/stream/dummy_helpers.h" +#include "FWCore/Framework/interface/stream/EDAnalyzer.h" +#include "FWCore/Framework/interface/stream/EDFilter.h" +#include "FWCore/Framework/interface/stream/EDProducer.h" + +#include +#include + +namespace edmtest { + + class TestAnalyzerWithInputProcessBlockAbility + : public edm::stream::EDAnalyzer, edm::InputProcessBlockCache> {}; + + class TestAnalyzerWithoutInputProcessBlockAbility : public edm::stream::EDAnalyzer> {}; + + class TestFilterWithInputProcessBlockAbility + : public edm::stream::EDFilter, edm::InputProcessBlockCache> {}; + + class TestFilterWithoutInputProcessBlockAbility : public edm::stream::EDFilter> {}; + + class TestProducerWithInputProcessBlockAbility + : public edm::stream::EDProducer, edm::InputProcessBlockCache> {}; + + class TestProducerWithoutInputProcessBlockAbility : public edm::stream::EDProducer> {}; + +} // namespace edmtest + +TEST_CASE("test InputProcessBlock", "[InputProcessBlock]") { + SECTION("test HasAbility") { + REQUIRE(edmtest::TestAnalyzerWithInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); + REQUIRE(!edmtest::TestAnalyzerWithoutInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); + } + SECTION("test type aliases") { + REQUIRE(std::is_same>()); + REQUIRE(std::is_same()); + REQUIRE(std::is_same::type, + std::unique_ptr>>()); + REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); + + REQUIRE(std::is_same>()); + REQUIRE(std::is_same()); + REQUIRE(std::is_same::type, + std::unique_ptr>>()); + REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); + + REQUIRE(std::is_same>()); + REQUIRE(std::is_same()); + REQUIRE(std::is_same::type, + std::unique_ptr>>()); + REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); + } +} diff --git a/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc b/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc new file mode 100644 index 0000000000000..5d2b225bf2109 --- /dev/null +++ b/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc @@ -0,0 +1,168 @@ +#include "catch.hpp" + +#include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" +#include "FWCore/Framework/interface/global/EDProducerBase.h" +#include "FWCore/Framework/interface/global/implementors.h" +#include "FWCore/Utilities/interface/StreamID.h" + +#include + +namespace edm { + class Event; + class EventSetup; +} // namespace edm + +namespace edmtest { + + class TestProcessBlockCacheA { + int x_; + }; + + class TestProcessBlockCacheB { + int x_; + }; + + class TestProcessBlockCacheC { + int x_; + }; + + class TestProcessBlockCacheD { + int x_; + }; + + class TestInputBlockCacheHolder0 + : public edm::global::impl::InputProcessBlockCacheHolder { + public: + bool wantsProcessBlocks() const override { return true; } + bool wantsInputProcessBlocks() const override { return true; } + bool wantsGlobalRuns() const override { return true; } + bool wantsGlobalLuminosityBlocks() const override { return true; } + bool wantsStreamRuns() const override { return true; } + bool wantsStreamLuminosityBlocks() const override { return true; } + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + }; + + class TestInputBlockCacheHolder1 + : public edm::global::impl::InputProcessBlockCacheHolder { + public: + bool wantsProcessBlocks() const override { return true; } + bool wantsInputProcessBlocks() const override { return true; } + bool wantsGlobalRuns() const override { return true; } + bool wantsGlobalLuminosityBlocks() const override { return true; } + bool wantsStreamRuns() const override { return true; } + bool wantsStreamLuminosityBlocks() const override { return true; } + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + }; + + class TestInputBlockCacheHolder2 : public edm::global::impl::InputProcessBlockCacheHolder { + public: + bool wantsProcessBlocks() const override { return true; } + bool wantsInputProcessBlocks() const override { return true; } + bool wantsGlobalRuns() const override { return true; } + bool wantsGlobalLuminosityBlocks() const override { return true; } + bool wantsStreamRuns() const override { return true; } + bool wantsStreamLuminosityBlocks() const override { return true; } + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + }; + + class TestInputBlockCacheHolder3 : public edm::global::impl::InputProcessBlockCacheHolder { + public: + bool wantsProcessBlocks() const override { return true; } + bool wantsInputProcessBlocks() const override { return true; } + bool wantsGlobalRuns() const override { return true; } + bool wantsGlobalLuminosityBlocks() const override { return true; } + bool wantsStreamRuns() const override { return true; } + bool wantsStreamLuminosityBlocks() const override { return true; } + void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override {} + }; +} // namespace edmtest + +TEST_CASE("test InputProcessBlockCacheHolder", "[InputProcessBlockCacheHolder]") { + SECTION("test countTypeInParameterPack") { + REQUIRE(edm::impl::countTypeInParameterPack() == 0); + REQUIRE(edm::impl::countTypeInParameterPack() == + 1); + REQUIRE(edm::impl::countTypeInParameterPack() == + 0); + REQUIRE(edm::impl::countTypeInParameterPack() == 1); + REQUIRE(edm::impl::countTypeInParameterPack() == 1); + REQUIRE(edm::impl::countTypeInParameterPack() == 2); + REQUIRE(edm::impl::countTypeInParameterPack() == 0); + + REQUIRE(edm::impl::countTypeInParameterPack() == 2); + REQUIRE(edm::impl::countTypeInParameterPack() == 1); + REQUIRE(edm::impl::countTypeInParameterPack() == 3); + REQUIRE(edm::impl::countTypeInParameterPack() == 2); + REQUIRE(edm::impl::countTypeInParameterPack() == 1); + REQUIRE(edm::impl::countTypeInParameterPack() == 0); + REQUIRE(edm::impl::countTypeInParameterPack() == 2); + REQUIRE(edm::impl::countTypeInParameterPack() == 1); + + // The following should not compile and I manually verified it doesn't + // REQUIRE(edm::impl::countTypeInParameterPack<>() == 0); + } + + SECTION("test indexInputProcessBlockCache") { + REQUIRE( + edm::impl::indexInputProcessBlockCache() == + 0); + REQUIRE(edm::impl::indexInputProcessBlockCache() == 0); + REQUIRE(edm::impl::indexInputProcessBlockCache() == 1); + REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); + // The following fails compilation if uncommented, tested manually + // REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); + // REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); + } + + SECTION("test constructor") { + edmtest::TestInputBlockCacheHolder0 holder0; + edmtest::TestInputBlockCacheHolder1 holder1; + edmtest::TestInputBlockCacheHolder2 holder2; + edmtest::TestInputBlockCacheHolder3 holder3; + } +} diff --git a/FWCore/Framework/test/test_module_delete_subprocess_cfg.py b/FWCore/Framework/test/test_module_delete_subprocess_cfg.py index 25be66a44bc97..9586325264ec8 100644 --- a/FWCore/Framework/test/test_module_delete_subprocess_cfg.py +++ b/FWCore/Framework/test/test_module_delete_subprocess_cfg.py @@ -168,7 +168,10 @@ def nonEventConsumer(transition, sourcePattern, expected): subprocessB.consumerEventFromA = intEventConsumer.clone(moduleLabel = "producerAEventConsumedInB", valueMustMatch = 1) subprocessB.consumerBeginLumiFromA = nonEventConsumer("beginLumi", "producerA%sConsumedInB", 2) subprocessB.consumerEndRunFromA = nonEventConsumer("endRun", "producerA%sConsumedInB", 5) -subprocessB.consumerBeginProcessBlockFromA = nonEventConsumer("beginProcessBlock", "producerA%sConsumedInB", 6) +subprocessB.consumerBeginProcessBlockFromA = intNonEventProducer.clone( + consumesAccessInputProcessBlock = cms.InputTag("producerABeginProcessBlockConsumedInB", "beginProcessBlock"), + expectAccessInputProcessBlock = cms.untracked.int32(6) +) subprocessB.consumerAEventNotConsumed = intGenericConsumer.clone( srcEvent = [ @@ -288,8 +291,14 @@ def nonEventConsumer(transition, sourcePattern, expected): subprocessBA.consumerEndLumiFromA = nonEventConsumer("endLumi", "producerA%sConsumedInBA", 30) subprocessBA.consumerBeginRunFromA = nonEventConsumer("beginRun", "producerA%sConsumedInBA", 40) subprocessBA.consumerEndRunFromA = nonEventConsumer("endRun", "producerA%sConsumedInBA", 50) -subprocessBA.consumerBeginProcessBlockFromA = nonEventConsumer("beginProcessBlock", "producerA%sConsumedInBA", 60) -subprocessBA.consumerEndProcessBlockFromA = nonEventConsumer("endProcessBlock", "producerA%sConsumedInBA", 70) +subprocessBA.consumerBeginProcessBlockFromA = intNonEventProducer.clone( + consumesAccessInputProcessBlock = cms.InputTag("producerABeginProcessBlockConsumedInBA", "beginProcessBlock"), + expectAccessInputProcessBlock = cms.untracked.int32(60) +) +subprocessBA.consumerEndProcessBlockFromA = intNonEventProducer.clone( + consumesAccessInputProcessBlock = cms.InputTag("producerAEndProcessBlockConsumedInBA", "endProcessBlock"), + expectAccessInputProcessBlock = cms.untracked.int32(70) +) subprocessBA.consumerABEventNotConsumed = intGenericConsumer.clone( srcEvent = [ @@ -389,7 +398,10 @@ def nonEventConsumer(transition, sourcePattern, expected): subprocessC.consumerEndLumiFromA = nonEventConsumer("endLumi", "producerA%sConsumedInC", 3) subprocessC.consumerBeginRunFromA = nonEventConsumer("beginRun", "producerA%sConsumedInC", 4) -subprocessC.consumerEndProcessBlockFromA = nonEventConsumer("endProcessBlock", "producerA%sConsumedInC", 7) +subprocessC.consumerEndProcessBlockFromA = intNonEventProducer.clone( + consumesAccessInputProcessBlock = cms.InputTag("producerAEndProcessBlockConsumedInC", "endProcessBlock"), + expectAccessInputProcessBlock = cms.untracked.int32(7) +) subprocessC.consumerAEventNotConsumed = intGenericConsumer.clone( srcEvent = [ diff --git a/FWCore/Integration/test/BuildFile.xml b/FWCore/Integration/test/BuildFile.xml index 467d8b2268185..9b420db88563b 100644 --- a/FWCore/Integration/test/BuildFile.xml +++ b/FWCore/Integration/test/BuildFile.xml @@ -77,6 +77,11 @@ + + + + + @@ -193,7 +198,7 @@ - + diff --git a/FWCore/Integration/test/PutOrMergeTestSource.cc b/FWCore/Integration/test/PutOrMergeTestSource.cc index 4a49359e8c693..b7f654e3ab458 100644 --- a/FWCore/Integration/test/PutOrMergeTestSource.cc +++ b/FWCore/Integration/test/PutOrMergeTestSource.cc @@ -33,7 +33,7 @@ namespace edmtest { ItemType getNextItemType() final; std::shared_ptr readRunAuxiliary_() final; std::shared_ptr readLuminosityBlockAuxiliary_() final; - std::unique_ptr readFile_() final; + std::shared_ptr readFile_() final; void readRun_(RunPrincipal& runPrincipal) final; void readEvent_(EventPrincipal& eventPrincipal) final; @@ -132,9 +132,9 @@ std::shared_ptr PutOrMergeTestSource::readRunAuxiliary_() { return id; } std::shared_ptr PutOrMergeTestSource::readLuminosityBlockAuxiliary_() { return {}; } -std::unique_ptr PutOrMergeTestSource::readFile_() { +std::shared_ptr PutOrMergeTestSource::readFile_() { ++stage_; - return std::make_unique(); + return std::make_shared(); } void PutOrMergeTestSource::readRun_(RunPrincipal& runPrincipal) { runAuxiliary()->setProcessHistoryID(historyID_); diff --git a/FWCore/Integration/test/TestFindProduct.cc b/FWCore/Integration/test/TestFindProduct.cc index 473bfd9d07cd3..6d0b1f6724933 100644 --- a/FWCore/Integration/test/TestFindProduct.cc +++ b/FWCore/Integration/test/TestFindProduct.cc @@ -1,25 +1,27 @@ -// This test module will look for IntProducts in Events. -// The number of IntProducts and their InputTags (label, -// instance, process) must be configured. +// This test module will try to get IntProducts in Events, +// Lumis, Runs and ProcessBlocks. The number of IntProducts +// and their InputTags (label, instance, process) must be configured. // One can also configure an expected value for the sum of // all the values in the IntProducts that are found. Note // that an IntProduct is just a test product that simply // contains a single integer. -// If the products are not found, then an exception is thrown. -// If the sum does not match there is an error message and -// an abort. +// If the expected products are not found or some other +// unexpected behavior occurs, an exception will be thrown. #include "DataFormats/Common/interface/Handle.h" +#include "DataFormats/Provenance/interface/Provenance.h" #include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Framework/interface/CacheHandle.h" #include "FWCore/Framework/interface/one/EDAnalyzer.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/getProducerParameterSet.h" #include "FWCore/Framework/interface/GetterOfProducts.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/moduleAbilities.h" #include "FWCore/Framework/interface/ProcessBlock.h" #include "FWCore/Framework/interface/ProcessMatch.h" #include "FWCore/Framework/interface/Run.h" @@ -28,13 +30,19 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/InputTag.h" #include "FWCore/Utilities/interface/EDGetToken.h" + +#include #include +#include +#include #include namespace edmtest { - class TestFindProduct - : public edm::one::EDAnalyzer { + class TestFindProduct : public edm::one::EDAnalyzer> { public: explicit TestFindProduct(edm::ParameterSet const& pset); virtual ~TestFindProduct(); @@ -45,12 +53,14 @@ namespace edmtest { void beginRun(edm::Run const&, edm::EventSetup const&) override; void endRun(edm::Run const&, edm::EventSetup const&) override; void beginProcessBlock(edm::ProcessBlock const&) override; + void accessInputProcessBlock(edm::ProcessBlock const&) override; void endProcessBlock(edm::ProcessBlock const&) override; void endJob() override; private: std::vector inputTags_; int expectedSum_; + int expectedCache_; int sum_; std::vector inputTagsNotFound_; bool getByTokenFirst_; @@ -62,6 +72,7 @@ namespace edmtest { std::vector inputTagsEndLumi_; std::vector inputTagsEndRun_; std::vector inputTagsBeginProcessBlock_; + std::vector inputTagsInputProcessBlock_; std::vector inputTagsEndProcessBlock_; std::vector inputTagsEndProcessBlock2_; std::vector inputTagsEndProcessBlock3_; @@ -74,6 +85,7 @@ namespace edmtest { std::vector> tokensEndLumi_; std::vector> tokensEndRun_; std::vector> tokensBeginProcessBlock_; + std::vector> tokensInputProcessBlock_; std::vector> tokensEndProcessBlock_; std::vector tokensEndProcessBlock2_; std::vector> tokensEndProcessBlock3_; @@ -90,6 +102,7 @@ namespace edmtest { TestFindProduct::TestFindProduct(edm::ParameterSet const& pset) : inputTags_(pset.getUntrackedParameter>("inputTags")), expectedSum_(pset.getUntrackedParameter("expectedSum", 0)), + expectedCache_(pset.getUntrackedParameter("expectedCache", 0)), sum_(0), inputTagsNotFound_(), getByTokenFirst_(pset.getUntrackedParameter("getByTokenFirst", false)), @@ -108,6 +121,8 @@ namespace edmtest { inputTagsEndRun_ = pset.getUntrackedParameter>("inputTagsEndRun", emptyTagVector); inputTagsBeginProcessBlock_ = pset.getUntrackedParameter>("inputTagsBeginProcessBlock", emptyTagVector); + inputTagsInputProcessBlock_ = + pset.getUntrackedParameter>("inputTagsInputProcessBlock", emptyTagVector); inputTagsEndProcessBlock_ = pset.getUntrackedParameter>("inputTagsEndProcessBlock", emptyTagVector); inputTagsEndProcessBlock2_ = @@ -138,6 +153,9 @@ namespace edmtest { for (auto const& tag : inputTagsBeginProcessBlock_) { tokensBeginProcessBlock_.push_back(consumes(tag)); } + for (auto const& tag : inputTagsInputProcessBlock_) { + tokensInputProcessBlock_.push_back(consumes(tag)); + } for (auto const& tag : inputTagsEndProcessBlock_) { tokensEndProcessBlock_.push_back(consumes(tag)); } @@ -150,11 +168,32 @@ namespace edmtest { for (auto const& tag : inputTagsEndProcessBlock4_) { tokensEndProcessBlock4_.push_back(consumes(tag)); } + + if (!tokensInputProcessBlock_.empty()) { + registerProcessBlockCacheFiller( + tokensInputProcessBlock_[0], + [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + for (auto const& token : tokensInputProcessBlock_) { + *returnValue += processBlock.get(token).value; + } + return returnValue; + }); + registerProcessBlockCacheFiller<1>( + tokensInputProcessBlock_[0], + [this](edm::ProcessBlock const& processBlock, std::shared_ptr const& previousCache) { + auto returnValue = std::make_shared(0); + for (auto const& token : tokensInputProcessBlock_) { + *returnValue += processBlock.get(token).value; + } + return returnValue; + }); + } } TestFindProduct::~TestFindProduct() {} - void TestFindProduct::analyze(edm::Event const& e, edm::EventSetup const&) { + void TestFindProduct::analyze(edm::Event const& event, edm::EventSetup const&) { edm::Handle h; edm::Handle hToken; edm::Handle> hView; @@ -164,35 +203,35 @@ namespace edmtest { for (std::vector::const_iterator iter = inputTags_.begin(), iEnd = inputTags_.end(); iter != iEnd; ++iter, ++iToken) { if (getByTokenFirst_) { - e.getByToken(*iToken, hToken); + event.getByToken(*iToken, hToken); *hToken; } - e.getByLabel(*iter, h); + event.getByLabel(*iter, h); sum_ += h->value; - e.getByToken(*iToken, hToken); + event.getByToken(*iToken, hToken); if (h->value != hToken->value) { - std::cerr << "TestFindProduct::analyze getByLabel and getByToken return inconsistent results " << std::endl; - abort(); + throw cms::Exception("TestFail") + << "TestFindProduct::analyze getByLabel and getByToken return inconsistent results"; } if (runProducerParameterCheck_) { - edm::ParameterSet const* producerPset = edm::getProducerParameterSet(*hToken.provenance(), e.processHistory()); + edm::ParameterSet const* producerPset = + edm::getProducerParameterSet(*hToken.provenance(), event.processHistory()); int par = producerPset->getParameter("ivalue"); // These expected values are just from knowing the values in the // configuration files for this test. int expectedParameterValue = 3; if (!iter->process().empty()) { - if (e.run() == 1) { + if (event.run() == 1) { expectedParameterValue = 1; } else { expectedParameterValue = 2; } } if (par != expectedParameterValue) { - std::cerr << "TestFindProduct::analyze unexpected value from producer parameter set" << std::endl; - abort(); + throw cms::Exception("TestFail") << "TestFindProduct::analyze unexpected value from producer parameter set"; } } } @@ -200,18 +239,18 @@ namespace edmtest { for (std::vector::const_iterator iter = inputTagsNotFound_.begin(), iEnd = inputTagsNotFound_.end(); iter != iEnd; ++iter, ++iToken) { - e.getByLabel(*iter, h); + event.getByLabel(*iter, h); if (h.isValid()) { - std::cerr << "TestFindProduct::analyze: getByLabel found a product that should not be found " - << h.provenance()->moduleLabel() << std::endl; - abort(); + throw cms::Exception("TestFail") + << "TestFindProduct::analyze: getByLabel found a product that should not be found " + << h.provenance()->moduleLabel(); } - e.getByToken(*iToken, hToken); + event.getByToken(*iToken, hToken); if (hToken.isValid()) { - std::cerr << "TestFindProduct::analyze: getByToken found a product that should not be found " - << hToken.provenance()->moduleLabel() << std::endl; - abort(); + throw cms::Exception("TestFail") + << "TestFindProduct::analyze: getByToken found a product that should not be found " + << hToken.provenance()->moduleLabel(); } } std::vector>>::const_iterator iTokenView = tokensView_.begin(); @@ -219,26 +258,44 @@ namespace edmtest { iter != iEnd; ++iter, ++iTokenView) { if (getByTokenFirst_) { - e.getByToken(*iTokenView, hViewToken); + event.getByToken(*iTokenView, hViewToken); *hViewToken; } - e.getByLabel(*iter, hView); + event.getByLabel(*iter, hView); sum_ += hView->at(0); - e.getByToken(*iTokenView, hViewToken); + event.getByToken(*iTokenView, hViewToken); if (hView->at(0) != hViewToken->at(0)) { - std::cerr << "TestFindProduct::analyze getByLabel and getByToken return inconsistent results " << std::endl; - abort(); + throw cms::Exception("TestFail") + << "TestFindProduct::analyze getByLabel and getByToken return inconsistent results"; } } // Get these also and add them into the sum edm::Handle h64; for (auto const& token : tokensUInt64_) { - e.getByToken(token, h64); + event.getByToken(token, h64); sum_ += h64->value; } + + if (expectedCache_ != 0) { + std::tuple, edm::CacheHandle> valueTuple = processBlockCaches(event); + { + edm::CacheHandle value = std::get<0>(valueTuple); + if (*value != expectedCache_) { + throw cms::Exception("TestFail") << "TestFindProduct::analyze 0 ProcessBlock cache has unexpected value " + << *value << " expected = " << expectedCache_; + } + } + { + edm::CacheHandle value = std::get<1>(valueTuple); + if (*value != expectedCache_) { + throw cms::Exception("TestFail") << "TestFindProduct::analyze 1 ProcessBlock cache has unexpected value " + << *value << " expected = " << expectedCache_; + } + } + } } void TestFindProduct::beginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) {} @@ -276,6 +333,13 @@ namespace edmtest { } } + void TestFindProduct::accessInputProcessBlock(edm::ProcessBlock const& processBlock) { + for (auto const& token : tokensInputProcessBlock_) { + int value = processBlock.get(token).value; + sum_ += value; + } + } + void TestFindProduct::endProcessBlock(edm::ProcessBlock const& processBlock) { std::vector values; for (auto const& token : tokensEndProcessBlock_) { @@ -322,6 +386,7 @@ namespace edmtest { << "TestFindProduct::endJob - Sum of test object values does not equal expected value"; } } + } // namespace edmtest using edmtest::TestFindProduct; diff --git a/FWCore/Integration/test/TestGlobalOutput.cc b/FWCore/Integration/test/TestGlobalOutput.cc index 79983a282dfba..eb9735e437307 100644 --- a/FWCore/Integration/test/TestGlobalOutput.cc +++ b/FWCore/Integration/test/TestGlobalOutput.cc @@ -9,6 +9,7 @@ #include "FWCore/ServiceRegistry/interface/Service.h" #include +#include #include namespace edm { @@ -23,6 +24,7 @@ namespace edm { void write(EventForOutput const& e) override; void writeLuminosityBlock(LuminosityBlockForOutput const&) override; void writeRun(RunForOutput const&) override; + void writeProcessBlock(ProcessBlockForOutput const&) override; void respondToOpenInputFile(FileBlock const&) override; void respondToCloseInputFile(FileBlock const&) override; @@ -32,42 +34,79 @@ namespace edm { std::shared_ptr globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const override; void globalEndLuminosityBlock(LuminosityBlockForOutput const&) const override; + void endJob() override; + + bool verbose_; + std::vector expectedProcessesWithProcessBlockProducts_; + int expectedWriteProcessBlockTransitions_; + int countWriteProcessBlockTransitions_ = 0; }; TestGlobalOutput::TestGlobalOutput(ParameterSet const& pset) : global::OutputModuleBase(pset), - global::OutputModule, LuminosityBlockCache>(pset) {} + global::OutputModule, LuminosityBlockCache>(pset), + verbose_(pset.getUntrackedParameter("verbose")), + expectedProcessesWithProcessBlockProducts_( + pset.getUntrackedParameter>("expectedProcessesWithProcessBlockProducts")), + expectedWriteProcessBlockTransitions_(pset.getUntrackedParameter("expectedWriteProcessBlockTransitions")) { + } TestGlobalOutput::~TestGlobalOutput() {} - void TestGlobalOutput::write(EventForOutput const& e) { LogAbsolute("TestGlobalOutput") << "global write event"; } + void TestGlobalOutput::write(EventForOutput const& e) { + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global write event"; + } + } void TestGlobalOutput::writeLuminosityBlock(LuminosityBlockForOutput const&) { - LogAbsolute("TestGlobalOutput") << "global writeLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global writeLuminosityBlock"; + } } void TestGlobalOutput::writeRun(RunForOutput const&) { LogAbsolute("TestGlobalOutput") << "global writeRun"; } + void TestGlobalOutput::writeProcessBlock(ProcessBlockForOutput const&) { + LogAbsolute("TestGlobalOutput") << "global writeProcessBlock"; + ++countWriteProcessBlockTransitions_; + if (!expectedProcessesWithProcessBlockProducts_.empty()) { + for (auto const& process : outputProcessBlockHelper().processesWithProcessBlockProducts()) { + LogAbsolute("TestGlobalOutput") << " " << process; + } + if (expectedProcessesWithProcessBlockProducts_ != + outputProcessBlockHelper().processesWithProcessBlockProducts()) { + throw cms::Exception("TestFailure") << "TestGlobalOutput::writeProcessBlock unexpected process name list"; + } + } + } + void TestGlobalOutput::respondToOpenInputFile(FileBlock const&) { - LogAbsolute("TestGlobalOutput") << "global respondToOpenInputFile"; + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global respondToOpenInputFile"; + } } void TestGlobalOutput::respondToCloseInputFile(FileBlock const&) { - LogAbsolute("TestGlobalOutput") << "global respondToCloseInputFile"; + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global respondToCloseInputFile"; + } } std::shared_ptr TestGlobalOutput::globalBeginRun(RunForOutput const&) const { LogAbsolute("TestGlobalOutput") << "global globalBeginRun"; - BranchIDLists const* theBranchIDLists = branchIDLists(); - for (auto const& branchIDList : *theBranchIDLists) { - LogAbsolute("TestGlobalOutput") << "A branchID list"; - for (auto const& branchID : branchIDList) { - LogAbsolute("TestGlobalOutput") << " global branchID " << branchID; + if (verbose_) { + BranchIDLists const* theBranchIDLists = branchIDLists(); + for (auto const& branchIDList : *theBranchIDLists) { + LogAbsolute("TestGlobalOutput") << "A branchID list"; + for (auto const& branchID : branchIDList) { + LogAbsolute("TestGlobalOutput") << " global branchID " << branchID; + } + } + edm::Service reg; + for (auto const& it : reg->productList()) { + LogAbsolute("TestGlobalOutput") << it.second; } - } - edm::Service reg; - for (auto const& it : reg->productList()) { - LogAbsolute("TestGlobalOutput") << it.second; } return std::make_shared(0); } @@ -77,17 +116,34 @@ namespace edm { } std::shared_ptr TestGlobalOutput::globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const { - LogAbsolute("TestGlobalOutput") << "global globalBeginLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global globalBeginLuminosityBlock"; + } return std::make_shared(0); } void TestGlobalOutput::globalEndLuminosityBlock(LuminosityBlockForOutput const&) const { - LogAbsolute("TestGlobalOutput") << "global globalEndLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestGlobalOutput") << "global globalEndLuminosityBlock"; + } + } + + void TestGlobalOutput::endJob() { + if (expectedWriteProcessBlockTransitions_ >= 0) { + if (expectedWriteProcessBlockTransitions_ != countWriteProcessBlockTransitions_) { + throw cms::Exception("TestFailure") + << "TestGlobalOutput::writeProcessBlock unexpected number of writeProcessBlock transitions"; + } + } } void TestGlobalOutput::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; OutputModule::fillDescription(desc); + desc.addUntracked("verbose", true); + desc.addUntracked>("expectedProcessesWithProcessBlockProducts", + std::vector()); + desc.addUntracked("expectedWriteProcessBlockTransitions", -1); descriptions.addDefault(desc); } } // namespace edm diff --git a/FWCore/Integration/test/TestLimitedOutput.cc b/FWCore/Integration/test/TestLimitedOutput.cc index f64d28c07b201..ab91944bcdf8b 100644 --- a/FWCore/Integration/test/TestLimitedOutput.cc +++ b/FWCore/Integration/test/TestLimitedOutput.cc @@ -9,6 +9,7 @@ #include "FWCore/ServiceRegistry/interface/Service.h" #include +#include #include namespace edm { @@ -23,6 +24,7 @@ namespace edm { void write(EventForOutput const& e) override; void writeLuminosityBlock(LuminosityBlockForOutput const&) override; void writeRun(RunForOutput const&) override; + void writeProcessBlock(ProcessBlockForOutput const&) override; void respondToOpenInputFile(FileBlock const&) override; void respondToCloseInputFile(FileBlock const&) override; @@ -32,42 +34,80 @@ namespace edm { std::shared_ptr globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const override; void globalEndLuminosityBlock(LuminosityBlockForOutput const&) const override; + + void endJob() override; + + bool verbose_; + std::vector expectedProcessesWithProcessBlockProducts_; + int expectedWriteProcessBlockTransitions_; + int countWriteProcessBlockTransitions_ = 0; }; TestLimitedOutput::TestLimitedOutput(ParameterSet const& pset) : limited::OutputModuleBase(pset), - limited::OutputModule, LuminosityBlockCache>(pset) {} + limited::OutputModule, LuminosityBlockCache>(pset), + verbose_(pset.getUntrackedParameter("verbose")), + expectedProcessesWithProcessBlockProducts_( + pset.getUntrackedParameter>("expectedProcessesWithProcessBlockProducts")), + expectedWriteProcessBlockTransitions_(pset.getUntrackedParameter("expectedWriteProcessBlockTransitions")) { + } TestLimitedOutput::~TestLimitedOutput() {} - void TestLimitedOutput::write(EventForOutput const& e) { LogAbsolute("TestLimitedOutput") << "limited write event"; } + void TestLimitedOutput::write(EventForOutput const& e) { + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited write event"; + } + } void TestLimitedOutput::writeLuminosityBlock(LuminosityBlockForOutput const&) { - LogAbsolute("TestLimitedOutput") << "limited writeLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited writeLuminosityBlock"; + } } void TestLimitedOutput::writeRun(RunForOutput const&) { LogAbsolute("TestLimitedOutput") << "limited writeRun"; } + void TestLimitedOutput::writeProcessBlock(ProcessBlockForOutput const&) { + LogAbsolute("TestLimitedOutput") << "limited writeProcessBlock"; + ++countWriteProcessBlockTransitions_; + if (!expectedProcessesWithProcessBlockProducts_.empty()) { + for (auto const& process : outputProcessBlockHelper().processesWithProcessBlockProducts()) { + LogAbsolute("TestLimitedOutput") << " " << process; + } + if (expectedProcessesWithProcessBlockProducts_ != + outputProcessBlockHelper().processesWithProcessBlockProducts()) { + throw cms::Exception("TestFailure") << "TestLimitedOutput::writeProcessBlock unexpected process name list"; + } + } + } + void TestLimitedOutput::respondToOpenInputFile(FileBlock const&) { - LogAbsolute("TestLimitedOutput") << "limited respondToOpenInputFile"; + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited respondToOpenInputFile"; + } } void TestLimitedOutput::respondToCloseInputFile(FileBlock const&) { - LogAbsolute("TestLimitedOutput") << "limited respondToCloseInputFile"; + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited respondToCloseInputFile"; + } } std::shared_ptr TestLimitedOutput::globalBeginRun(RunForOutput const&) const { LogAbsolute("TestLimitedOutput") << "limited globalBeginRun"; - BranchIDLists const* theBranchIDLists = branchIDLists(); - for (auto const& branchIDList : *theBranchIDLists) { - LogAbsolute("TestLimitedOutput") << "A branchID list"; - for (auto const& branchID : branchIDList) { - LogAbsolute("TestLimitedOutput") << " limited branchID " << branchID; + if (verbose_) { + BranchIDLists const* theBranchIDLists = branchIDLists(); + for (auto const& branchIDList : *theBranchIDLists) { + LogAbsolute("TestLimitedOutput") << "A branchID list"; + for (auto const& branchID : branchIDList) { + LogAbsolute("TestLimitedOutput") << " limited branchID " << branchID; + } + } + edm::Service reg; + for (auto const& it : reg->productList()) { + LogAbsolute("TestLimitedOutput") << it.second; } - } - edm::Service reg; - for (auto const& it : reg->productList()) { - LogAbsolute("TestLimitedOutput") << it.second; } return std::make_shared(0); } @@ -77,17 +117,34 @@ namespace edm { } std::shared_ptr TestLimitedOutput::globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const { - LogAbsolute("TestLimitedOutput") << "limited globalBeginLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited globalBeginLuminosityBlock"; + } return std::make_shared(0); } void TestLimitedOutput::globalEndLuminosityBlock(LuminosityBlockForOutput const&) const { - LogAbsolute("TestLimitedOutput") << "limited globalEndLuminosityBlock"; + if (verbose_) { + LogAbsolute("TestLimitedOutput") << "limited globalEndLuminosityBlock"; + } + } + + void TestLimitedOutput::endJob() { + if (expectedWriteProcessBlockTransitions_ >= 0) { + if (expectedWriteProcessBlockTransitions_ != countWriteProcessBlockTransitions_) { + throw cms::Exception("TestFailure") + << "TestLimitedOutput::writeProcessBlock unexpected number of writeProcessBlock transitions"; + } + } } void TestLimitedOutput::fillDescriptions(ConfigurationDescriptions& descriptions) { ParameterSetDescription desc; OutputModule::fillDescription(desc); + desc.addUntracked("verbose", true); + desc.addUntracked>("expectedProcessesWithProcessBlockProducts", + std::vector()); + desc.addUntracked("expectedWriteProcessBlockTransitions", -1); descriptions.addDefault(desc); } } // namespace edm diff --git a/FWCore/Integration/test/TestOneOutput.cc b/FWCore/Integration/test/TestOneOutput.cc new file mode 100644 index 0000000000000..671d330e292a2 --- /dev/null +++ b/FWCore/Integration/test/TestOneOutput.cc @@ -0,0 +1,412 @@ + +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/Framework/interface/one/OutputModule.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/Framework/interface/ConstProductRegistry.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/Utilities/interface/Exception.h" + +#include "TTree.h" + +#include +#include +#include + +namespace edm { + + class TestOneOutput : public one::OutputModule, LuminosityBlockCache> { + public: + explicit TestOneOutput(ParameterSet const& pset); + ~TestOneOutput() override; + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + private: + void write(EventForOutput const& e) override; + void writeLuminosityBlock(LuminosityBlockForOutput const&) override; + void writeRun(RunForOutput const&) override; + void writeProcessBlock(ProcessBlockForOutput const&) override; + + void respondToOpenInputFile(FileBlock const&) override; + void respondToCloseInputFile(FileBlock const&) override; + void testFileBlock(FileBlock const&); + + std::shared_ptr globalBeginRun(RunForOutput const&) const override; + void globalEndRun(RunForOutput const&) override; + + std::shared_ptr globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const override; + void globalEndLuminosityBlock(LuminosityBlockForOutput const&) override; + void endJob() override; + + bool verbose_; + std::vector expectedProcessesWithProcessBlockProducts_; + std::vector expectedTopProcessesWithProcessBlockProducts_; + std::vector expectedAddedProcesses_; + std::vector expectedTopAddedProcesses_; + std::vector expectedTopCacheIndices0_; + std::vector expectedTopCacheIndices1_; + std::vector expectedTopCacheIndices2_; + std::vector expectedProcessNamesAtWrite_; + int expectedWriteProcessBlockTransitions_; + unsigned int countWriteProcessBlockTransitions_ = 0; + unsigned int countInputFiles_ = 0; + bool requireNullTTreesInFileBlock_; + bool testTTreesInFileBlock_; + std::vector expectedCacheIndexSize_; + unsigned int expectedProcessesInFirstFile_; + std::vector expectedCacheIndexVectorsPerFile_; + std::vector expectedNEntries0_; + std::vector expectedNEntries1_; + std::vector expectedNEntries2_; + std::vector expectedCacheEntriesPerFile0_; + std::vector expectedCacheEntriesPerFile1_; + std::vector expectedCacheEntriesPerFile2_; + std::vector expectedOuterOffset_; + std::vector expectedTranslateFromStoredIndex_; + unsigned int expectedNAddedProcesses_; + bool expectedProductsFromInputKept_; + }; + + TestOneOutput::TestOneOutput(ParameterSet const& pset) + : one::OutputModuleBase(pset), + one::OutputModule, LuminosityBlockCache>(pset), + verbose_(pset.getUntrackedParameter("verbose")), + expectedProcessesWithProcessBlockProducts_( + pset.getUntrackedParameter>("expectedProcessesWithProcessBlockProducts")), + expectedTopProcessesWithProcessBlockProducts_( + pset.getUntrackedParameter>("expectedTopProcessesWithProcessBlockProducts")), + expectedAddedProcesses_(pset.getUntrackedParameter>("expectedAddedProcesses")), + expectedTopAddedProcesses_(pset.getUntrackedParameter>("expectedTopAddedProcesses")), + expectedTopCacheIndices0_(pset.getUntrackedParameter>("expectedTopCacheIndices0")), + expectedTopCacheIndices1_(pset.getUntrackedParameter>("expectedTopCacheIndices1")), + expectedTopCacheIndices2_(pset.getUntrackedParameter>("expectedTopCacheIndices2")), + expectedProcessNamesAtWrite_( + pset.getUntrackedParameter>("expectedProcessNamesAtWrite")), + expectedWriteProcessBlockTransitions_(pset.getUntrackedParameter("expectedWriteProcessBlockTransitions")), + requireNullTTreesInFileBlock_(pset.getUntrackedParameter("requireNullTTreesInFileBlock")), + testTTreesInFileBlock_(pset.getUntrackedParameter("testTTreesInFileBlock")), + expectedCacheIndexSize_(pset.getUntrackedParameter>("expectedCacheIndexSize")), + expectedProcessesInFirstFile_(pset.getUntrackedParameter("expectedProcessesInFirstFile")), + expectedCacheIndexVectorsPerFile_( + pset.getUntrackedParameter>("expectedCacheIndexVectorsPerFile")), + expectedNEntries0_(pset.getUntrackedParameter>("expectedNEntries0")), + expectedNEntries1_(pset.getUntrackedParameter>("expectedNEntries1")), + expectedNEntries2_(pset.getUntrackedParameter>("expectedNEntries2")), + expectedCacheEntriesPerFile0_( + pset.getUntrackedParameter>("expectedCacheEntriesPerFile0")), + expectedCacheEntriesPerFile1_( + pset.getUntrackedParameter>("expectedCacheEntriesPerFile1")), + expectedCacheEntriesPerFile2_( + pset.getUntrackedParameter>("expectedCacheEntriesPerFile2")), + expectedOuterOffset_(pset.getUntrackedParameter>("expectedOuterOffset")), + expectedTranslateFromStoredIndex_( + pset.getUntrackedParameter>("expectedTranslateFromStoredIndex")), + expectedNAddedProcesses_(pset.getUntrackedParameter("expectedNAddedProcesses")), + expectedProductsFromInputKept_(pset.getUntrackedParameter("expectedProductsFromInputKept")) {} + + TestOneOutput::~TestOneOutput() {} + + void TestOneOutput::write(EventForOutput const& e) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one write event"; + } + } + + void TestOneOutput::writeLuminosityBlock(LuminosityBlockForOutput const&) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one writeLuminosityBlock"; + } + } + + void TestOneOutput::writeRun(RunForOutput const&) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one writeRun"; + } + } + + void TestOneOutput::writeProcessBlock(ProcessBlockForOutput const& pb) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one writeProcessBlock"; + } + if (countWriteProcessBlockTransitions_ < expectedProcessNamesAtWrite_.size()) { + if (expectedProcessNamesAtWrite_[countWriteProcessBlockTransitions_] != pb.processName()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected process name"; + } + } + if (!expectedProcessesWithProcessBlockProducts_.empty()) { + if (verbose_) { + for (auto const& process : outputProcessBlockHelper().processesWithProcessBlockProducts()) { + LogAbsolute("TestOneOutput") << " " << process; + } + } + if (expectedProcessesWithProcessBlockProducts_ != + outputProcessBlockHelper().processesWithProcessBlockProducts()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected process name list"; + } + } + if (!(!expectedAddedProcesses_.empty() && expectedAddedProcesses_[0] == "DONOTTEST")) { + if (expectedAddedProcesses_ != outputProcessBlockHelper().processBlockHelper()->addedProcesses()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected addedProcesses list"; + } + } + if (!(!expectedTopAddedProcesses_.empty() && expectedTopAddedProcesses_[0] == "DONOTTEST")) { + if (expectedTopAddedProcesses_ != + outputProcessBlockHelper().processBlockHelper()->topProcessBlockHelper()->addedProcesses()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected top addedProcesses list"; + } + } + if (!expectedTopProcessesWithProcessBlockProducts_.empty()) { + // Same test as the previous except check the list of names in + // the top level process name list from the EventProcessor + if (expectedTopProcessesWithProcessBlockProducts_ != outputProcessBlockHelper() + .processBlockHelper() + ->topProcessBlockHelper() + ->topProcessesWithProcessBlockProducts()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected top process name list"; + } + // Almost the same as the previous test, should get the same result + if (expectedTopProcessesWithProcessBlockProducts_ != + outputProcessBlockHelper().processBlockHelper()->topProcessesWithProcessBlockProducts()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected top process name list 2"; + } + + std::vector const* expectedTopCacheIndices = nullptr; + if (countInputFiles_ == 1 && !expectedTopCacheIndices0_.empty()) { + expectedTopCacheIndices = &expectedTopCacheIndices0_; + } else if (countInputFiles_ == 2 && !expectedTopCacheIndices1_.empty()) { + expectedTopCacheIndices = &expectedTopCacheIndices1_; + } else if (countInputFiles_ == 3 && !expectedTopCacheIndices2_.empty()) { + expectedTopCacheIndices = &expectedTopCacheIndices2_; + } + if (expectedTopCacheIndices != nullptr) { + unsigned int expectedInputProcesses = + expectedTopProcessesWithProcessBlockProducts_.size() - + outputProcessBlockHelper().processBlockHelper()->topProcessBlockHelper()->addedProcesses().size(); + + std::vector> const& topProcessBlockCacheIndices = + outputProcessBlockHelper().processBlockHelper()->processBlockCacheIndices(); + if (expectedTopCacheIndices->size() != expectedInputProcesses * topProcessBlockCacheIndices.size()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected sizes related to top cache indices on input file " + << (countInputFiles_ - 1); + } + unsigned int iStored = 0; + for (unsigned int i = 0; i < topProcessBlockCacheIndices.size(); ++i) { + if (topProcessBlockCacheIndices[i].size() != expectedInputProcesses) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected size of inner cache indices vector on input file " + << (countInputFiles_ - 1); + } + for (unsigned int j = 0; j < topProcessBlockCacheIndices[i].size(); ++j) { + if (topProcessBlockCacheIndices[i][j] != (*expectedTopCacheIndices)[iStored]) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected cache index value on input file " + << (countInputFiles_ - 1); + } + ++iStored; + } + } + } + } + if (expectedProcessesInFirstFile_ != 0) { + if (expectedProcessesInFirstFile_ != outputProcessBlockHelper().processBlockHelper()->nProcessesInFirstFile()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for nProcessesInFirstFile"; + } + } + for (unsigned int i = 0; i < expectedCacheIndexVectorsPerFile_.size(); ++i) { + if (i < countInputFiles_) { + if (expectedCacheIndexVectorsPerFile_[i] != + outputProcessBlockHelper().processBlockHelper()->cacheIndexVectorsPerFile()[i]) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for cacheIndexVectorsPerFile, element " << i; + } + } + } + if (countInputFiles_ >= 1 && !expectedNEntries0_.empty()) { + if (expectedNEntries0_ != outputProcessBlockHelper().processBlockHelper()->nEntries()[0]) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected value for nEntries 0"; + } + } + if (countInputFiles_ >= 2 && !expectedNEntries1_.empty()) { + if (expectedNEntries1_ != outputProcessBlockHelper().processBlockHelper()->nEntries()[1]) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected value for nEntries 1"; + } + } + if (countInputFiles_ >= 3 && !expectedNEntries2_.empty()) { + if (expectedNEntries2_ != outputProcessBlockHelper().processBlockHelper()->nEntries()[2]) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected value for nEntries 2"; + } + } + + if (countInputFiles_ == 1 && !expectedCacheEntriesPerFile0_.empty()) { + if (expectedCacheEntriesPerFile0_ != outputProcessBlockHelper().processBlockHelper()->cacheEntriesPerFile()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for cacheEntriesPerFile 0"; + } + } else if (countInputFiles_ == 2 && !expectedCacheEntriesPerFile1_.empty()) { + if (expectedCacheEntriesPerFile1_ != outputProcessBlockHelper().processBlockHelper()->cacheEntriesPerFile()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for cacheEntriesPerFile 1"; + } + } else if (countInputFiles_ == 3 && !expectedCacheEntriesPerFile2_.empty()) { + if (expectedCacheEntriesPerFile2_ != outputProcessBlockHelper().processBlockHelper()->cacheEntriesPerFile()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for cacheEntriesPerFile 2"; + } + } + + if (!expectedOuterOffset_.empty() && (countInputFiles_ - 1) < expectedOuterOffset_.size()) { + if (expectedOuterOffset_[countInputFiles_ - 1] != + outputProcessBlockHelper().processBlockHelper()->outerOffset()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for outerOffset, file " << (countInputFiles_ - 1); + } + } + if (countWriteProcessBlockTransitions_ < expectedCacheIndexSize_.size()) { + if (expectedCacheIndexSize_[countWriteProcessBlockTransitions_] != + outputProcessBlockHelper().processBlockHelper()->processBlockCacheIndices().size()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected cache index size " + << outputProcessBlockHelper().processBlockHelper()->processBlockCacheIndices().size(); + } + } + if (!expectedTranslateFromStoredIndex_.empty()) { + if (expectedTranslateFromStoredIndex_ != outputProcessBlockHelper().translateFromStoredIndex()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for translateFromStoredIndex"; + } + } + if (expectedNAddedProcesses_ != 0xffffffff) { + if (expectedNAddedProcesses_ != outputProcessBlockHelper().nAddedProcesses()) { + throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected value for nAddedProcesses"; + } + } + if (expectedProductsFromInputKept_ != outputProcessBlockHelper().productsFromInputKept()) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected value for productsFromInputKept"; + } + ++countWriteProcessBlockTransitions_; + } + + void TestOneOutput::respondToOpenInputFile(FileBlock const& fb) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one respondToOpenInputFile"; + } + testFileBlock(fb); + ++countInputFiles_; + } + + void TestOneOutput::respondToCloseInputFile(FileBlock const& fb) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one respondToCloseInputFile"; + } + testFileBlock(fb); + } + + void TestOneOutput::testFileBlock(FileBlock const& fb) { + if (requireNullTTreesInFileBlock_) { + if (fb.tree() != nullptr || fb.lumiTree() != nullptr || fb.runTree() != nullptr || + !fb.processBlockTrees().empty() || !fb.processesWithProcessBlockTrees().empty() || + fb.processBlockTree(std::string("DoesNotExist")) != nullptr) { + throw cms::Exception("TestFailure") + << "TestOneOutput::respondToOpenInputFile expected null TTree pointers in FileBlock"; + } + } else if (testTTreesInFileBlock_) { + if (fb.tree() == nullptr || std::string("Events") != fb.tree()->GetName() || + std::string("LuminosityBlocks") != fb.lumiTree()->GetName() || + std::string("Runs") != fb.runTree()->GetName() || fb.processesWithProcessBlockTrees().size() != 2 || + fb.processesWithProcessBlockTrees()[0] != "PROD1" || fb.processesWithProcessBlockTrees()[1] != "MERGE" || + fb.processBlockTrees().size() != 2 || + std::string("ProcessBlocksPROD1") != fb.processBlockTrees()[0]->GetName() || + std::string("ProcessBlocksMERGE") != fb.processBlockTrees()[1]->GetName() || + std::string("ProcessBlocksPROD1") != fb.processBlockTree("PROD1")->GetName() || + std::string("ProcessBlocksMERGE") != fb.processBlockTree("MERGE")->GetName() || + fb.processBlockTree("DOESNOTEXIST") != nullptr) { + throw cms::Exception("TestFailure") << "TestOneOutput::testFileBlock failed. Testing tree pointers"; + } + } + } + + std::shared_ptr TestOneOutput::globalBeginRun(RunForOutput const&) const { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one globalBeginRun"; + edm::Service reg; + for (auto const& it : reg->productList()) { + LogAbsolute("TestOneOutput") << it.second; + } + } + return std::make_shared(0); + } + + void TestOneOutput::globalEndRun(RunForOutput const&) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one globalEndRun"; + } + } + + std::shared_ptr TestOneOutput::globalBeginLuminosityBlock(LuminosityBlockForOutput const&) const { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one globalBeginLuminosityBlock"; + } + return std::make_shared(0); + } + + void TestOneOutput::globalEndLuminosityBlock(LuminosityBlockForOutput const&) { + if (verbose_) { + LogAbsolute("TestOneOutput") << "one globalEndLuminosityBlock"; + } + } + + void TestOneOutput::endJob() { + if (expectedWriteProcessBlockTransitions_ >= 0) { + if (static_cast(expectedWriteProcessBlockTransitions_) != countWriteProcessBlockTransitions_) { + throw cms::Exception("TestFailure") + << "TestOneOutput::writeProcessBlock unexpected number of writeProcessBlock transitions"; + } + } + } + + void TestOneOutput::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + OutputModule::fillDescription(desc); + desc.addUntracked("verbose", true); + desc.addUntracked>("expectedProcessesWithProcessBlockProducts", + std::vector()); + desc.addUntracked>("expectedTopProcessesWithProcessBlockProducts", + std::vector()); + desc.addUntracked>("expectedAddedProcesses", + std::vector(1, std::string("DONOTTEST"))); + desc.addUntracked>("expectedTopAddedProcesses", + std::vector(1, std::string("DONOTTEST"))); + desc.addUntracked>("expectedTopCacheIndices0", std::vector()); + desc.addUntracked>("expectedTopCacheIndices1", std::vector()); + desc.addUntracked>("expectedTopCacheIndices2", std::vector()); + desc.addUntracked>("expectedProcessNamesAtWrite", std::vector()); + desc.addUntracked("expectedWriteProcessBlockTransitions", -1); + desc.addUntracked("requireNullTTreesInFileBlock", false); + desc.addUntracked("testTTreesInFileBlock", false); + desc.addUntracked>("expectedCacheIndexSize", std::vector()); + desc.addUntracked("expectedProcessesInFirstFile", 0); + desc.addUntracked>("expectedCacheIndexVectorsPerFile", std::vector()); + desc.addUntracked>("expectedNEntries0", std::vector()); + desc.addUntracked>("expectedNEntries1", std::vector()); + desc.addUntracked>("expectedNEntries2", std::vector()); + desc.addUntracked>("expectedCacheEntriesPerFile0", std::vector()); + desc.addUntracked>("expectedCacheEntriesPerFile1", std::vector()); + desc.addUntracked>("expectedCacheEntriesPerFile2", std::vector()); + desc.addUntracked>("expectedOuterOffset", std::vector()); + desc.addUntracked>("expectedTranslateFromStoredIndex", std::vector()); + desc.addUntracked("expectedNAddedProcesses", 0xffffffff); + desc.addUntracked("expectedProductsFromInputKept", true); + + descriptions.addDefault(desc); + } +} // namespace edm + +using edm::TestOneOutput; +DEFINE_FWK_MODULE(TestOneOutput); diff --git a/FWCore/Integration/test/ThrowingSource.cc b/FWCore/Integration/test/ThrowingSource.cc index a9010263b1cb9..373348197b289 100644 --- a/FWCore/Integration/test/ThrowingSource.cc +++ b/FWCore/Integration/test/ThrowingSource.cc @@ -16,7 +16,7 @@ namespace edm { void endJob() override; void beginLuminosityBlock(edm::LuminosityBlock&) override; void beginRun(edm::Run&) override; - std::unique_ptr readFile_() override; + std::shared_ptr readFile_() override; void closeFile_() override; std::shared_ptr readRunAuxiliary_() override; std::shared_ptr readLuminosityBlockAuxiliary_() override; @@ -83,10 +83,10 @@ namespace edm { throw cms::Exception("TestThrow") << "ThrowingSource::beginRun"; } - std::unique_ptr ThrowingSource::readFile_() { + std::shared_ptr ThrowingSource::readFile_() { if (whenToThrow_ == kReadFile) throw cms::Exception("TestThrow") << "ThrowingSource::readFile_"; - return std::make_unique(); + return std::make_shared(); } void ThrowingSource::closeFile_() { diff --git a/FWCore/Integration/test/run_TestOutput.sh b/FWCore/Integration/test/run_TestOutput.sh index 403e7addb4d2d..34b3af2eed8ca 100755 --- a/FWCore/Integration/test/run_TestOutput.sh +++ b/FWCore/Integration/test/run_TestOutput.sh @@ -14,6 +14,7 @@ pushd ${LOCAL_TMP_DIR} grep "global write event" testOutput1.log > /dev/null || die "grep failed to find 'global write event'" $? grep "global writeLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'global writeLuminosityBlock'" $? grep "global writeRun" testOutput1.log > /dev/null || die "grep failed to find 'global writeRun'" $? + grep "global writeProcessBlock" testOutput1.log > /dev/null || die "grep failed to find 'global writeProcessBlock'" $? grep "global respondToOpenInputFile" testOutput1.log > /dev/null || die "grep failed to find 'global respondToOpenInputFile'" $? grep "global respondToCloseInputFile" testOutput1.log > /dev/null || die "grep failed to find 'global respondToCloseInputFile'" $? grep "global globalBeginRun" testOutput1.log > /dev/null || die "grep failed to find 'global globalBeginRun'" $? @@ -28,6 +29,7 @@ pushd ${LOCAL_TMP_DIR} grep "limited write event" testOutput1.log > /dev/null || die "grep failed to find 'limited write event'" $? grep "limited writeLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited writeLuminosityBlock'" $? grep "limited writeRun" testOutput1.log > /dev/null || die "grep failed to find 'limited writeRun'" $? + grep "limited writeProcessBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited writeProcessBlock'" $? grep "limited respondToOpenInputFile" testOutput1.log > /dev/null || die "grep failed to find 'limited respondToOpenInputFile'" $? grep "limited respondToCloseInputFile" testOutput1.log > /dev/null || die "grep failed to find 'limited respondToCloseInputFile'" $? grep "limited globalBeginRun" testOutput1.log > /dev/null || die "grep failed to find 'limited globalBeginRun'" $? @@ -43,6 +45,7 @@ pushd ${LOCAL_TMP_DIR} grep "global write event" testOutput2.log > /dev/null || die "grep failed to find 'global write event'" $? grep "global writeLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'global writeLuminosityBlock'" $? grep "global writeRun" testOutput2.log > /dev/null || die "grep failed to find 'global writeRun'" $? + grep "global writeProcessBlock" testOutput2.log > /dev/null || die "grep failed to find 'global writeProcessBlock'" $? grep "global respondToOpenInputFile" testOutput2.log > /dev/null || die "grep failed to find 'global respondToOpenInputFile'" $? grep "global respondToCloseInputFile" testOutput2.log > /dev/null || die "grep failed to find 'global respondToCloseInputFile'" $? grep "global globalBeginRun" testOutput2.log > /dev/null || die "grep failed to find 'global globalBeginRun'" $? @@ -54,6 +57,7 @@ pushd ${LOCAL_TMP_DIR} grep "limited write event" testOutput2.log > /dev/null || die "grep failed to find 'limited write event'" $? grep "limited writeLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited writeLuminosityBlock'" $? grep "limited writeRun" testOutput2.log > /dev/null || die "grep failed to find 'limited writeRun'" $? + grep "limited writeProcessBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited writeProcessBlock'" $? grep "limited respondToOpenInputFile" testOutput2.log > /dev/null || die "grep failed to find 'limited respondToOpenInputFile'" $? grep "limited respondToCloseInputFile" testOutput2.log > /dev/null || die "grep failed to find 'limited respondToCloseInputFile'" $? grep "limited globalBeginRun" testOutput2.log > /dev/null || die "grep failed to find 'limited globalBeginRun'" $? diff --git a/FWCore/Integration/test/run_TestProcessBlock.sh b/FWCore/Integration/test/run_TestProcessBlock.sh new file mode 100755 index 0000000000000..9bca5dcdff707 --- /dev/null +++ b/FWCore/Integration/test/run_TestProcessBlock.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } + +pushd ${LOCAL_TMP_DIR} + + echo "testProcessBlock1" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock1_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock1_cfg.py" $? + + # The MetaData ProcessBlock branch and the TTree should exist to hold the ProcessBlock + # data. The Events branch should not exist because there were not any ProcessBlock branches + # saved from an input file. Test that here: + edmFileUtil -l -t MetaData -P file:testProcessBlock1.root > testProcessBlock1ContentsM.txt + grep "Branch.* ProcessBlockHelper " testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? + grep "TTree.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? + edmFileUtil -t Events -P file:testProcessBlock1.root > testProcessBlock1ContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" $? + + echo "testProcessBlock2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock2_cfg.py" $? + + echo "testProcessBlock3" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock3_cfg.py" $? + + echo "testProcessBlock4" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock4_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock4_cfg.py" $? + + echo "testProcessBlockMerge" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge_cfg.py" $? + + # The ProcessBlock Branches and TTrees should exist in this case. Test that here: + edmFileUtil -l -t MetaData -P file:testProcessBlockMerge.root > testProcessBlockMContentsM.txt + grep "Branch.* ProcessBlockHelper " testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? + grep "TTree.*ProcessBlocksPROD1" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? + grep "TTree.*ProcessBlocksMERGE" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksMERGE TTree" $? + edmFileUtil -t Events -P file:testProcessBlockMerge.root > testProcessBlockMContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlockMContentsE.txt || die "Check for existence of eventToProcessBlockIndexes branch" $? + + echo "testProcessBlockTEST" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockTEST_cfg.py || die "cmsRun testProcessBlockTEST_cfg.py" $? + + echo "testProcessBlockRead" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockRead_cfg.py &> testProcessBlockRead.log || die "cmsRun testProcessBlockRead_cfg.py" $? + grep "InputProcessBlockIntAnalyzer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntAnalyzer::accessInputProcessBlock was called" $? + grep "InputProcessBlockIntFilter::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntFilter::accessInputProcessBlock was called" $? + grep "InputProcessBlockIntProducer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntProducer::accessInputProcessBlock was called" $? + + echo "testProcessBlock2Dropped" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2Dropped_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock2Dropped_cfg.py" $? + + # The ProcessBlock Branches and TTrees should not exist in this case because + # all the ProcessBlock products are dropped. Test that here: + edmFileUtil -l -t MetaData -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsM.txt + grep "Branch.* ProcessBlockHelper " testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlockHelper branch" $? + grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" $? + edmFileUtil -t Events -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" $? + + # This one intentionally fails because the product content of the + # files does not match (strict merging requirements for ProcessBlocks) + echo "testProcessBlockFailMerge" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py > /dev/null 2>&1 && die "cmsRun testProcessBlockFailMerge_cfg.py" $? + + echo "testProcessBlockMerge2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge2_cfg.py" $? + + echo "testProcessBlockMergeOfMergedFiles" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMergeOfMergedFiles_cfg.py" $? + + echo "testProcessBlockNOMergeOfMergedFiles" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNOMergeOfMergedFiles_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNOMergeOfMergedFiles_cfg.py" $? + + echo "testProcessBlockRead2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockRead2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockRead2_cfg.py" $? + + echo "testProcessBlockSubProcess" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcess_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcess_cfg.py" $? + + echo "testProcessBlockSubProcessRead1" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead1_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessRead1_cfg.py" $? + + echo "testProcessBlockSubProcessRead2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessRead2_cfg.py" $? + + echo "testProcessBlockSubProcessLooper" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessLooper_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessLooper_cfg.py" $? + + echo "testProcessBlock5" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock5_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock5_cfg.py" $? + + echo "testProcessBlockMerge3" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge3_cfg.py" $? + + echo "testProcessBlockMergeOfMergedFiles2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMergeOfMergedFiles2_cfg.py" $? + + echo "testProcessBlockDropOnInput" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnInput_cfg.py" $? + + echo "testProcessBlockThreeFileInput" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockThreeFileInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockThreeFileInput_cfg.py" $? + + echo "testProcessBlockReadThreeFileInput" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadThreeFileInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadThreeFileInput_cfg.py" $? + + echo "testLooperEventNavigation2" + cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation2_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation2.txt > /dev/null 2>&1 || die "cmsRun testLooperEventNavigation2_cfg.py" $? + + echo "testLooperEventNavigation3" + cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation3_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation3.txt > /dev/null 2>&1 || die "cmsRun testLooperEventNavigation3_cfg.py" $? + + echo "testProcessBlockDropOnOutput" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnOutput_cfg.py" $? + + echo "testProcessBlockDropOnOutput2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnOutput2_cfg.py" $? + + echo "testProcessBlockReadDropOnOutput" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadDropOnOutput_cfg.py" $? + + echo "testProcessBlockReadDropOnOutput2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadDropOnOutput2_cfg.py" $? + + # The next three tests would be relevant if we disabled the strict merging requirement + # in ProductRegistry.cc for ProcessBlock products (a one line code change). As long + # as we always enforce the strict merging requirement these tests will fail, but they + # would be useful if we decide to allow that requirement to be disabled in the future. + # I ran them manually with the ProductRegistry.cc modified to disable the requirement + # and in May 2021 these tests passed. + + #echo "testProcessBlockNonStrict" + #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict_cfg.py" $? + + #echo "testProcessBlockNonStrict2" + #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict2_cfg.py" $? + + #echo "testProcessBlockNonStrict3" + #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict3_cfg.py" $? + + rm testProcessBlock1ContentsM.txt + rm testProcessBlock1ContentsE.txt + rm testProcessBlock2DroppedContentsM.txt + rm testProcessBlock2DroppedContentsE.txt + rm testProcessBlockMContentsM.txt + rm testProcessBlockMContentsE.txt + rm testProcessBlockRead.log + + rm testProcessBlock1.root + rm testProcessBlock2.root + rm testProcessBlock3.root + rm testProcessBlock4.root + rm testProcessBlockMerge.root + rm testProcessBlockTest.root + rm testProcessBlockRead.root + rm testProcessBlock2Dropped.root + rm testProcessBlockFailMerge.root + rm testProcessBlockMerge2.root + rm testProcessBlockMergeOfMergedFiles.root + rm testProcessBlockNOMergeOfMergedFiles.root + rm testProcessBlockRead2.root + rm testProcessBlockSubProcessTest.root + rm testProcessBlockSubProcessRead.root + rm testProcessBlockSubProcessReadAgain.root + rm testProcessBlockSubProcessRead1.root + rm testProcessBlockSubProcessRead2.root + rm testProcessBlockSubProcessLooperTest.root + rm testProcessBlockSubProcessLooperRead.root + rm testProcessBlockSubProcessLooperReadAgain.root + rm testProcessBlock5.root + rm testProcessBlockMerge3.root + rm testProcessBlockMergeOfMergedFiles2.root + rm testProcessBlockDropOnInput.root + rm testProcessBlockThreeFileInput.root + rm testProcessBlockReadThreeFileInput.root + rm testProcessBlockDropOnOutput.root + rm testProcessBlockDropOnOutput2.root + rm testProcessBlockDropOnOutput2_2.root + rm testProcessBlockReadDropOnOutput.root + rm testProcessBlockReadDropOnOutput2.root +popd + +exit 0 diff --git a/FWCore/Integration/test/testConsumesInfo_cfg.py b/FWCore/Integration/test/testConsumesInfo_cfg.py index 72052957e8970..14383ca93cbef 100644 --- a/FWCore/Integration/test/testConsumesInfo_cfg.py +++ b/FWCore/Integration/test/testConsumesInfo_cfg.py @@ -442,25 +442,38 @@ copyProcess.processBlockTest1 = cms.EDAnalyzer("TestFindProduct", inputTags = cms.untracked.VInputTag(), - expectedSum = cms.untracked.int32(460034), + expectedSum = cms.untracked.int32(450034), + # This does not work in a SubProcess because the accessInputProcessBlock + # transition does not occur until after event processing is over + # expectedCache = cms.untracked.int32(110000), inputTagsBeginProcessBlock = cms.untracked.VInputTag( cms.InputTag("intProducerBeginProcessBlock"), - cms.InputTag("intProducerBeginProcessBlock", "", "PROD1"), cms.InputTag("intProducerBeginProcessBlock", "", "COPY") ), inputTagsEndProcessBlock = cms.untracked.VInputTag( cms.InputTag("intProducerBeginProcessBlock"), - cms.InputTag("intProducerBeginProcessBlock", "", "PROD1"), cms.InputTag("intProducerBeginProcessBlock", "", "COPY"), cms.InputTag("intProducerEndProcessBlock"), - cms.InputTag("intProducerEndProcessBlock", "", "PROD1"), cms.InputTag("intProducerEndProcessBlock", "", "COPY"), cms.InputTag("intProducerEndProcessBlock", "", cms.InputTag.currentProcess()) + ), + inputTagsInputProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerBeginProcessBlock", "", "PROD1"), + cms.InputTag("intProducerEndProcessBlock", "", "PROD1") ) ) +copyProcess.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'COPY'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(2), + expectedProductsFromInputKept = cms.untracked.bool(False) +) + copyProcess.path3 = cms.Path( copyProcess.intProducerBeginProcessBlock * copyProcess.intProducerEndProcessBlock * copyProcess.processBlockTest1 ) + +copyProcess.endPath = cms.EndPath(copyProcess.testOneOutput) diff --git a/FWCore/Integration/test/testGetBy1Mod_cfg.py b/FWCore/Integration/test/testGetBy1Mod_cfg.py index c83a62717efa8..41c48a0a9074b 100644 --- a/FWCore/Integration/test/testGetBy1Mod_cfg.py +++ b/FWCore/Integration/test/testGetBy1Mod_cfg.py @@ -19,6 +19,12 @@ process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(2)) -process.t = cms.Task(process.intProducer) +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(10000)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(100000)) + +process.t = cms.Task(process.intProducer, + process.intProducerBeginProcessBlock, + process.intProducerEndProcessBlock ) process.e = cms.EndPath(process.out, process.t) diff --git a/FWCore/Integration/test/testGetBy3_cfg.py b/FWCore/Integration/test/testGetBy3_cfg.py index a654a85ba37a2..c8aa52d20b073 100644 --- a/FWCore/Integration/test/testGetBy3_cfg.py +++ b/FWCore/Integration/test/testGetBy3_cfg.py @@ -173,6 +173,13 @@ expectedSum = cms.untracked.int32(93) ) +process.a2000 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsInputProcessBlock = cms.untracked.VInputTag( cms.InputTag("intProducerBeginProcessBlock"), cms.InputTag("intProducerEndProcessBlock")), + expectedSum = cms.untracked.int32(110000), + expectedCache = cms.untracked.int32(110000) +) + process.p = cms.Path(process.intProducer * process.a1 * process.a2 * process.a3 * process.a4 * process.a5 * process.a6) process.p0 = cms.Path(process.a10 * process.a20 * process.a30 * process.a40 * process.a50 * process.a60 * process.a70) @@ -198,6 +205,8 @@ process.p1005 = cms.Path(process.a1005 * process.a1006) +process.p2000 = cms.Path(process.a2000) + process.t = cms.Task(process.intProducerU, process.intProducerA, process.nonProducer, process.intVectorSetProducer, process.intVectorProducer) diff --git a/FWCore/Integration/test/testLooperEventNavigation2.txt b/FWCore/Integration/test/testLooperEventNavigation2.txt new file mode 100644 index 0000000000000..77448094fdfcb --- /dev/null +++ b/FWCore/Integration/test/testLooperEventNavigation2.txt @@ -0,0 +1,24 @@ +0 +0 +0 +2 +3 +1 +2 +0 +0 +0 +0 +2 +1 +1 +1 +2 +3 +1 +1 +2 +2 +1 +2 +4 diff --git a/FWCore/Integration/test/testLooperEventNavigation2_cfg.py b/FWCore/Integration/test/testLooperEventNavigation2_cfg.py new file mode 100644 index 0000000000000..d5cb16155cbda --- /dev/null +++ b/FWCore/Integration/test/testLooperEventNavigation2_cfg.py @@ -0,0 +1,41 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +# 57 transitions = 12 events + 15 access input ProcessBlock transitions + 30 fill calls +# sum 16442 = 2 x (3300 + 4400 + 444) + 3 x (11 + 22 + 44) = 16288 + 231 = +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(57), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(16519) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(16), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 6, 6, 6, 8, 8, 8, 10, 10, 10) +) + +process.looper = cms.Looper("NavigateEventsLooper") + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('file:testLooperEventNavigation2.root'), + fastCloning = cms.untracked.bool(False) +) + +process.path1 = cms.Path(process.readProcessBlocksOneAnalyzer) +process.endpath1 = cms.EndPath(process.out * process.testOneOutput) diff --git a/FWCore/Integration/test/testLooperEventNavigation3.txt b/FWCore/Integration/test/testLooperEventNavigation3.txt new file mode 100644 index 0000000000000..4567460864833 --- /dev/null +++ b/FWCore/Integration/test/testLooperEventNavigation3.txt @@ -0,0 +1,6 @@ +1 +1 +1 +1 +1 +4 diff --git a/FWCore/Integration/test/testLooperEventNavigation3_cfg.py b/FWCore/Integration/test/testLooperEventNavigation3_cfg.py new file mode 100644 index 0000000000000..089b6ebbca527 --- /dev/null +++ b/FWCore/Integration/test/testLooperEventNavigation3_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) + , skipEvents = cms.untracked.uint32(7) +) + +# 24 transitions = 6 events + 6 access input ProcessBlock transitions + 12 fill calls +# sum 8221 = 3300 + 4400 + 444 + 11 + 22 + 44 = 7700 + 444 + 77 = +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4) +) + +process.looper = cms.Looper("NavigateEventsLooper") + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('file:testLooperEventNavigation2.root'), + fastCloning = cms.untracked.bool(False) +) + +process.path1 = cms.Path(process.readProcessBlocksOneAnalyzer) +process.endpath1 = cms.EndPath(process.out * process.testOneOutput) diff --git a/FWCore/Integration/test/testLooperEventNavigation_cfg.py b/FWCore/Integration/test/testLooperEventNavigation_cfg.py index 4ddd4ec8855c5..4632c3efaa2e5 100644 --- a/FWCore/Integration/test/testLooperEventNavigation_cfg.py +++ b/FWCore/Integration/test/testLooperEventNavigation_cfg.py @@ -6,11 +6,6 @@ process.MessageLogger.cerr.FwkReport.reportEvery = 1000 process.MessageLogger.cerr.threshold = 'ERROR' -import FWCore.Framework.test.cmsExceptionsFatalOption_cff -process.options = cms.untracked.PSet( - Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow -) - process.source = cms.Source("PoolSource", fileNames = cms.untracked.vstring( 'file:testRunMerge1.root', diff --git a/FWCore/Integration/test/testProcessBlock1_cfg.py b/FWCore/Integration/test/testProcessBlock1_cfg.py new file mode 100644 index 0000000000000..fed2d7ade5f06 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock1_cfg.py @@ -0,0 +1,98 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock1.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + requireNullTTreesInFileBlock = cms.untracked.bool(True), + expectedProductsFromInputKept = cms.untracked.bool(False), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(1)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(10)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50)) + +process.testEDAliasOriginal = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(111)) + +process.testEDAliasAlias = cms.EDAlias(testEDAliasOriginal = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + expectedSum = cms.untracked.int32(111), + inputTags = cms.untracked.VInputTag(), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("testEDAliasOriginal") + ) +) + +process.a2 = cms.EDAnalyzer("TestFindProduct", + expectedSum = cms.untracked.int32(111), + inputTags = cms.untracked.VInputTag(), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("testEDAliasAlias") + ) +) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.testEDAliasOriginal * + process.a1 * + process.a2 +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlock2Dropped_cfg.py b/FWCore/Integration/test/testProcessBlock2Dropped_cfg.py new file mode 100644 index 0000000000000..c06408a2584ed --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock2Dropped_cfg.py @@ -0,0 +1,74 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(2), + firstLuminosityBlock = cms.untracked.uint32(1), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock2Dropped.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ), + expectedProductsFromInputKept = cms.untracked.bool(False) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(1)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(10)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlock2_cfg.py b/FWCore/Integration/test/testProcessBlock2_cfg.py new file mode 100644 index 0000000000000..e667edd899ac5 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock2_cfg.py @@ -0,0 +1,45 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(2) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock2.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(2)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(20)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(7)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(70)) + +process.testEDAliasOriginal = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(111)) + +process.testEDAliasAlias = cms.EDAlias(testEDAliasOriginal = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.testEDAliasOriginal +) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlock3_cfg.py b/FWCore/Integration/test/testProcessBlock3_cfg.py new file mode 100644 index 0000000000000..0c60a76076df2 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock3_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(3) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock3.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(300)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(3000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(30000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(300000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlock4_cfg.py b/FWCore/Integration/test/testProcessBlock4_cfg.py new file mode 100644 index 0000000000000..0b35cb2fb98f8 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock4_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(4) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock4.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(400)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(4000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(40000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(400000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlock5_cfg.py b/FWCore/Integration/test/testProcessBlock5_cfg.py new file mode 100644 index 0000000000000..226bc8c300f5a --- /dev/null +++ b/FWCore/Integration/test/testProcessBlock5_cfg.py @@ -0,0 +1,38 @@ +import FWCore.ParameterSet.Config as cms + +# Intentionally reversing the order of process names in this series +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(5) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlock5.root') +) + + +process.intProducerBeginProcessBlockMM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(404)) + +process.intProducerEndProcessBlockMM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(440)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockMM * + process.intProducerEndProcessBlockMM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py b/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py new file mode 100644 index 0000000000000..dd872ebbc3ae3 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py @@ -0,0 +1,82 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), + inputCommands=cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(37), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440), + consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), + consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), + consumesProcessBlockNotFound3 = cms.InputTag("intProducerBeginProcessBlockM"), + consumesProcessBlockNotFound4 = cms.InputTag("intProducerEndProcessBlockM") +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(28), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(0), + consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), + consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), + consumesProcessBlockNotFound3 = cms.InputTag("intProducerBeginProcessBlockM"), + consumesProcessBlockNotFound4 = cms.InputTag("intProducerEndProcessBlockM") +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnInput.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_*_READ" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 1, 4, 2, 4, 3, 4), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 1, 4, 2, 4, 3, 4, 5, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8), + expectedProcessesInFirstFile = cms.untracked.uint32(2), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4), + expectedNEntries0 = cms.untracked.vuint32(4, 1), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(5) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockDropOnOutput2_cfg.py b/FWCore/Integration/test/testProcessBlockDropOnOutput2_cfg.py new file mode 100644 index 0000000000000..1355ed4e86388 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockDropOnOutput2_cfg.py @@ -0,0 +1,92 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("DROPONOUTPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), +) + +process.intProducerBeginProcessBlockN = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5)) + +process.intProducerEndProcessBlockN = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50)) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(0, 2, 3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedProductsFromInputKept = cms.untracked.bool(True) +) + +process.out2 = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput2_2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ) +) + +process.testOneOutput2 = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedProductsFromInputKept = cms.untracked.bool(False) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockN * process.intProducerEndProcessBlockN) + +process.e = cms.EndPath( + process.out * + process.testOneOutput * + process.out2 * + process.testOneOutput2 +) diff --git a/FWCore/Integration/test/testProcessBlockDropOnOutput_cfg.py b/FWCore/Integration/test/testProcessBlockDropOnOutput_cfg.py new file mode 100644 index 0000000000000..a2882f1a0e285 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockDropOnOutput_cfg.py @@ -0,0 +1,53 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("DROPONOUTPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), +) + + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(0, 2), + expectedNAddedProcesses = cms.untracked.uint32(0), + expectedProductsFromInputKept = cms.untracked.bool(True), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9) +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockFailMerge_cfg.py b/FWCore/Integration/test/testProcessBlockFailMerge_cfg.py new file mode 100644 index 0000000000000..027ed7d7d8a28 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockFailMerge_cfg.py @@ -0,0 +1,32 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +# The second file has ProcessBlock products dropped that +# were in the first file so this should fail the strict +# merge requirements on ProcessBlock products. +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock1.root', + 'file:testProcessBlock2Dropped.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockFailMerge.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(2)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(20)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlockMerge2_cfg.py b/FWCore/Integration/test/testProcessBlockMerge2_cfg.py new file mode 100644 index 0000000000000..54b79cf34d153 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockMerge2_cfg.py @@ -0,0 +1,59 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) + #fileMode = cms.untracked.string('NOMERGE') +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock3.root', + 'file:testProcessBlock4.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge2.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(204)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(240)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(208)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(280)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockMerge3_cfg.py b/FWCore/Integration/test/testProcessBlockMerge3_cfg.py new file mode 100644 index 0000000000000..b64785d000ab9 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockMerge3_cfg.py @@ -0,0 +1,43 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock5.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge3.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('MERGEOFMERGED', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(2) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(8)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(80)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles2_cfg.py b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles2_cfg.py new file mode 100644 index 0000000000000..a4d671a049ba9 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles2_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +# Intentionally reversing the order of process names in this series +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge3.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockMergeOfMergedFiles2.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(707)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(7000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(40000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(400000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py new file mode 100644 index 0000000000000..15a6f0f754c18 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py @@ -0,0 +1,77 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(30), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockMergeOfMergedFiles.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring('MERGEOFMERGED'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 2, 1, 2), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 2, 1, 2, 3, 5, 4, 5) +) + +process.intProducerBeginProcessBlockMM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(304)) + +process.intProducerEndProcessBlockMM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(340)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockMM * + process.intProducerEndProcessBlockMM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockMerge_cfg.py b/FWCore/Integration/test/testProcessBlockMerge_cfg.py new file mode 100644 index 0000000000000..dff8a69d3d0a7 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockMerge_cfg.py @@ -0,0 +1,87 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock1.root', + 'file:testProcessBlock2.root' + ) +) + +# transitions 14 = 6 events + 2 InputProcessBlock transitions + 3 x 2 cache filling calls +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(14), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7700), + expectedSum = cms.int32(33) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasAlias_*_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedTopAddedProcesses = cms.untracked.vstring('MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedTopCacheIndices0 = cms.untracked.vuint32(0), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 1) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(8)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(80)) + +process.a2000 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsInputProcessBlock = cms.untracked.VInputTag( cms.InputTag("testEDAliasAlias")), + expectedSum = cms.untracked.int32(222), + expectedCache = cms.untracked.int32(111) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer1 * + process.a2000 +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py b/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py new file mode 100644 index 0000000000000..9c6bc83d7fb87 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py @@ -0,0 +1,72 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1), + fileMode = cms.untracked.string('NOMERGE') +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(30), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockNOMergeOfMergedFiles.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 2, 2, 2, 2, 2) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(304)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(340)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py new file mode 100644 index 0000000000000..eb7dec9550ae8 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py @@ -0,0 +1,83 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST2") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockNonStrict.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck"), + inputCommands=cms.untracked.vstring( + 'keep *', + 'drop *_*_*_NONSTRICTTEST' + ) +) + +# 117 transitions = 45 events + 27 InputProcessBlock transitions + (3 x 15) Cache filling transitions +# sum = (11 + 22 + 3300 + 4400 + 7707) x 3 + (44 + 44 + 444) x 2 = 47384 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(117), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(47384) +) + +# 90 transitions = 45 events + 27 InputProcessBlock transitions + (3 x 6) Cache filling transitions +# sum = (44 + 44 + 444) x 2 = 1064 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(90), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(1064) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 20, 23, 11, 20, 23, 12, 21, 23, 13, 21, 23, 14, 22, 24, 15, 4294967295, 25, 16, 4294967295, 25, 17, 4294967295, 25, 18, 4294967295, 25, 19, 4294967295, 26), + expectedWriteProcessBlockTransitions = cms.untracked.int32(28), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 10), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(10, 3, 4), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 17), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py new file mode 100644 index 0000000000000..93b8047ffed5b --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py @@ -0,0 +1,86 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockDropOnInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck"), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ) +) + +# 40 transitions = 30 events + 10 InputProcessBlock transitions + (3 x 0) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(40), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + #expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(532) +) + +# 40 transitions = 30 events + 10 InputProcessBlock transitions + (3 x 0) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(40), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + #expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(532) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 4294967295, 4294967295, 4294967295), + expectedWriteProcessBlockTransitions = cms.untracked.int32(11), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 1), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(0, 0, 0), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 0), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py new file mode 100644 index 0000000000000..72b46ceef0f87 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py @@ -0,0 +1,79 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockDropOnInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +# 77 transitions = 30 events + 17 InputProcessBlock transitions + (3 x 10) Cache filling transitions +# sum = (11 + 22 + 3300 + 4400 + 7707) x 2 + 44 + 444 + 44 = 31412 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(77), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(31412) +) + +# 59 transitions = 30 events + 17 InputProcessBlock transitions + (3 x 4) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(59), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(532) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 4294967295, 15, 11, 4294967295, 15, 12, 4294967295, 15, 13, 4294967295, 15, 14, 4294967295, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 5), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(5, 0, 2), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 7), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockRead2_cfg.py b/FWCore/Integration/test/testProcessBlockRead2_cfg.py new file mode 100644 index 0000000000000..6b491d77f4062 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockRead2_cfg.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root' + ) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(31), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockRead2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + + +process.p = cms.Path(process.transientIntProducerEndProcessBlock * process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py b/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py new file mode 100644 index 0000000000000..f839d56780718 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockDropOnOutput2.root' + ) +) + +# 38 = 15 events + 8 access input ProcessBlock transitions + 15 fillCache functor calls +# sum 15440 = 11 + 22 + 3300 + 4400 + 7707 +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(38), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockReadDropOnOutput2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 5, 7, 1, 5, 7, 2, 5, 7, 3, 5, 7, 4, 6, 7), + expectedWriteProcessBlockTransitions = cms.untracked.int32(9) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py b/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py new file mode 100644 index 0000000000000..17556de910b06 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockDropOnOutput.root' + ) +) + +# 37 = 15 events + 7 access input ProcessBlock transitions + 15 fillCache functor calls +# sum 15440 = 11 + 22 + 3300 + 4400 + 7707 +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(37), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockReadDropOnOutput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 5, 1, 5, 2, 5, 3, 5, 4, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py b/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py new file mode 100644 index 0000000000000..c59d65fca545f --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py @@ -0,0 +1,55 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockThreeFileInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(71), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(24193) +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(53), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(1020) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockReadThreeFileInput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 9, 14, 1, 9, 14, 2, 10, 14, 3, 10, 14, 4, 11, 15, 5, 12, 16, 6, 12, 16, 7, 13, 16, 8, 13, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(9), + expectedNEntries0 = cms.untracked.vuint32(9, 5, 3), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(17), + expectedOuterOffset = cms.untracked.vuint32(0) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockRead_cfg.py b/FWCore/Integration/test/testProcessBlockRead_cfg.py new file mode 100644 index 0000000000000..12da104b98780 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockRead_cfg.py @@ -0,0 +1,277 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockTest.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockRead.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.intProducerBeginProcessBlockR = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(100)) + +process.intProducerEndProcessBlockR = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(1000)) + +process.readProcessBlocks = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzer", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(0, 11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksG = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksNS = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerNS") +process.readProcessBlocksGNS = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerGNS") + +process.readProcessBlocksStreamFilter = cms.EDFilter("edmtest::stream::InputProcessBlockIntFilter", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(0, 11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksGStreamFilter = cms.EDFilter("edmtest::stream::InputProcessBlockIntFilterG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksStreamProducer = cms.EDProducer("edmtest::stream::InputProcessBlockIntProducer", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(0, 11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksGStreamProducer = cms.EDProducer("edmtest::stream::InputProcessBlockIntProducerG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlock") +) + +process.readProcessBlocksOneFilter = cms.EDFilter("edmtest::one::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksOneProducer = cms.EDProducer("edmtest::one::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalAnalyzer = cms.EDAnalyzer("edmtest::global::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalFilter = cms.EDFilter("edmtest::global::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalProducer = cms.EDProducer("edmtest::global::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksLimitedAnalyzer = cms.EDAnalyzer("edmtest::limited::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksLimitedFilter = cms.EDFilter("edmtest::limited::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksLimitedProducer = cms.EDProducer("edmtest::limited::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksGlobalAnalyzerNoRegistration = cms.EDAnalyzer("edmtest::global::InputProcessBlockIntAnalyzerNoRegistration", + transitions = cms.int32(6), +) + +process.readProcessBlocksDoesNotExist = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(9), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlock1 = cms.InputTag("doesNotExist", ""), + consumesEndProcessBlock1 = cms.InputTag("doesNotExist", ""), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun0 = cms.vint32(0, 11, 22), + expectedByRun1 = cms.vint32(), + expectedByRun2 = cms.vint32(0, 44, 44) +) + +process.readProcessBlocksExplicitProcess = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(10), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlockB", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlockB", ""), + consumesBeginProcessBlock1 = cms.InputTag("intProducerBeginProcessBlockB", "", "PROD1"), + consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlockB", "", "PROD1"), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockB", "", "MERGE"), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockB", "", "MERGE"), + expectedByRun0 = cms.vint32(0, 88, 88), + expectedByRun1 = cms.vint32(0, 55, 77), + expectedByRun2 = cms.vint32(0, 88, 88) +) + +process.readProcessBlocksExplicitProcess2 = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(11), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlock1 = cms.InputTag("intProducerBeginProcessBlock", "", "PROD1"), + consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlock", "", "PROD1"), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", "", "MERGE"), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", "", "MERGE"), + expectedByRun0 = cms.vint32(0, 11, 22), + expectedByRun1 = cms.vint32(0, 11, 22), + expectedByRun2 = cms.vint32(0, 44, 44) +) + +process.readProcessBlocksReuseCache = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerReuseCache", + transitions = cms.int32(8), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(0, 11, 11) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockR * + process.intProducerEndProcessBlockR * + process.readProcessBlocks * + process.readProcessBlocksG * + process.readProcessBlocksNS * + process.readProcessBlocksGNS * + process.readProcessBlocksStreamFilter * + process.readProcessBlocksGStreamFilter * + process.readProcessBlocksStreamProducer * + process.readProcessBlocksGStreamProducer * + process.readProcessBlocksOneAnalyzer * + process.readProcessBlocksOneFilter * + process.readProcessBlocksOneProducer * + process.readProcessBlocksGlobalAnalyzer * + process.readProcessBlocksGlobalFilter * + process.readProcessBlocksGlobalProducer * + process.readProcessBlocksLimitedAnalyzer * + process.readProcessBlocksLimitedFilter * + process.readProcessBlocksLimitedProducer * + process.readProcessBlocksDoesNotExist * + process.readProcessBlocksGlobalAnalyzerNoRegistration * + process.readProcessBlocksExplicitProcess * + process.readProcessBlocksExplicitProcess2 * + process.readProcessBlocksReuseCache +) + +process.e = cms.EndPath( + process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py new file mode 100644 index 0000000000000..c9d06094586b3 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py @@ -0,0 +1,219 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.essource = cms.ESSource("EmptyESSource", + recordName = cms.string('DummyRecord'), + iovIsRunNotTime = cms.bool(True), + firstValid = cms.vuint32(1) +) +process.add_(cms.ESProducer("LoadableDummyProvider", + value = cms.untracked.int32(5))) + +process.looper = cms.Looper("IntTestLooper", + expectESValue = cms.untracked.int32(5) +) + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root' + ) +) + +process.intProducerBeginProcessBlockT = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4000)) + +process.intProducerEndProcessBlockT = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40000)) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessLooperTest.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(24), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.eventIntProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.nonEventIntProducer = cms.EDProducer("NonEventIntProducer", + ivalue = cms.int32(1) +) + +process.p = cms.Path( + process.eventIntProducer * + process.transientIntProducerEndProcessBlock * + process.nonEventIntProducer * + process.intProducerBeginProcessBlockT * + process.intProducerEndProcessBlockT +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) + +readProcess = cms.Process("READ") +process.addSubProcess(cms.SubProcess(readProcess, + outputCommands = cms.untracked.vstring( + "keep *" + ) +)) + +readProcess.intProducerBeginProcessBlockR = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5000)) + +readProcess.intProducerEndProcessBlockR = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50000)) + +readProcess.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessLooperRead.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readProcess.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(27), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readProcess.p = cms.Path( + readProcess.intProducerBeginProcessBlockR * + readProcess.intProducerEndProcessBlockR +) + +readProcess.e = cms.EndPath( + readProcess.out * + readProcess.testOneOutput +) + +readAgainProcess = cms.Process("READAGAIN") +readProcess.addSubProcess(cms.SubProcess(readAgainProcess, + outputCommands = cms.untracked.vstring( + "keep *" + ) +)) + +readAgainProcess.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(99), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(24663) +) + +readAgainProcess.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(81), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 44, 44, 444, 444), + expectedSum = cms.int32(1464) +) + +readAgainProcess.readProcessBlocksOneAnalyzer3 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(72), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644), + expectedSum = cms.int32(1464) +) + +readAgainProcess.readProcessBlocksOneAnalyzer4 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(72), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockT", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + # The expectedByRun test cannot work because the data is from an earlier SubProcess + expectedByRun = cms.vint32(), + expectedFillerSum = cms.untracked.int32(396000), + expectedSum = cms.int32(1464) +) + +readAgainProcess.readProcessBlocksOneAnalyzer5 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(72), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockR", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + # The expectedByRun test cannot work because the data is from an earlier SubProcess + expectedByRun = cms.vint32(), + expectedFillerSum = cms.untracked.int32(495000), + expectedSum = cms.int32(1464), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockT"), + expectedCacheSize = cms.untracked.uint32(7) +) + +readAgainProcess.intProducerBeginProcessBlockRA = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(100000)) + +readAgainProcess.intProducerEndProcessBlockRA = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(1000000)) + +readAgainProcess.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessLooperReadAgain.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readAgainProcess.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ', 'READAGAIN'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(30), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readAgainProcess.p = cms.Path( + readAgainProcess.intProducerBeginProcessBlockRA * + readAgainProcess.intProducerEndProcessBlockRA * + readAgainProcess.readProcessBlocksOneAnalyzer1 * + readAgainProcess.readProcessBlocksOneAnalyzer2 * + readAgainProcess.readProcessBlocksOneAnalyzer3 * + readAgainProcess.readProcessBlocksOneAnalyzer4 * + readAgainProcess.readProcessBlocksOneAnalyzer5 +) + +readAgainProcess.e = cms.EndPath( + readAgainProcess.out * + readAgainProcess.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py new file mode 100644 index 0000000000000..6ba4501dfb72d --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py @@ -0,0 +1,76 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READSUBPROCESSOUTPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockSubProcessRead.root' + ) +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(33), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221), + expectedFillerSum = cms.untracked.int32(23199), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlock") +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockT", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 44000, 44000, 44000, 44000), + expectedSum = cms.int32(488), + expectedFillerSum = cms.untracked.int32(132000), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockT") +) + +process.readProcessBlocksOneAnalyzer3 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockR", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 55000, 55000, 55000, 55000), + expectedSum = cms.int32(488), + expectedFillerSum = cms.untracked.int32(165000), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockR"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockR") +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessRead1.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(10) +) + +process.p = cms.Path( + process.readProcessBlocksOneAnalyzer1 * + process.readProcessBlocksOneAnalyzer2 * + process.readProcessBlocksOneAnalyzer3 +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py new file mode 100644 index 0000000000000..bc4ee6ad11723 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py @@ -0,0 +1,93 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READSUBPROCESSOUTPUT2") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockSubProcessReadAgain.root' + ) +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(34), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221), + expectedFillerSum = cms.untracked.int32(23199), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlock") +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(25), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockT", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 44000, 44000, 44000, 44000), + expectedSum = cms.int32(488), + expectedFillerSum = cms.untracked.int32(132000), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockT") +) + +process.readProcessBlocksOneAnalyzer3 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(25), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockR", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 55000, 55000, 55000, 55000), + expectedSum = cms.int32(488), + expectedFillerSum = cms.untracked.int32(165000), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockR"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockR") +) + +process.readProcessBlocksOneAnalyzer4 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(25), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockRA", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockRA", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 1100000, 1100000, 1100000, 1100000), + expectedSum = cms.int32(488), + expectedFillerSum = cms.untracked.int32(3300000), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockRA"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockRA") +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessRead2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ', 'READAGAIN'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ', 'READAGAIN'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 7, 8, 9, 1, 4, 6, 7, 8, 9, 2, 5, 6, 7, 8, 9, 3, 5, 6, 7, 8, 9), + expectedWriteProcessBlockTransitions = cms.untracked.int32(11) +) + +process.p = cms.Path( + process.readProcessBlocksOneAnalyzer1 * + process.readProcessBlocksOneAnalyzer2 * + process.readProcessBlocksOneAnalyzer3 * + process.readProcessBlocksOneAnalyzer4 +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py new file mode 100644 index 0000000000000..29bc47b1b7046 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py @@ -0,0 +1,227 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root' + ) +) + +process.intProducerBeginProcessBlockT = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4000)) + +process.intProducerEndProcessBlockT = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40000)) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessTest.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.eventIntProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.nonEventIntProducer = cms.EDProducer("NonEventIntProducer", + ivalue = cms.int32(1) +) + +process.p = cms.Path( + process.eventIntProducer * + process.transientIntProducerEndProcessBlock * + process.nonEventIntProducer * + process.intProducerBeginProcessBlockT * + process.intProducerEndProcessBlockT +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) + +readProcess = cms.Process("READ") +process.addSubProcess(cms.SubProcess(readProcess, + outputCommands = cms.untracked.vstring( + "keep *" + ) +)) + +readProcess.intProducerBeginProcessBlockR = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5000)) + +readProcess.intProducerEndProcessBlockR = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50000)) + +readProcess.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessRead.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readProcess.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(9), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readProcess.p = cms.Path( + readProcess.intProducerBeginProcessBlockR * + readProcess.intProducerEndProcessBlockR +) + +readProcess.e = cms.EndPath( + readProcess.out * + readProcess.testOneOutput +) + +readAgainProcess = cms.Process("READAGAIN") +readProcess.addSubProcess(cms.SubProcess(readAgainProcess, + outputCommands = cms.untracked.vstring( + "keep *" + ) +)) + +# transitions = 12 events + 9 access input ProcessBlock transitions + 12 fill cache functor calls +# sum = 11 + 22 + 3300 + 4400 + 44 + 444 +readAgainProcess.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(33), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +# transitions = 12 events + 9 access input ProcessBlock transitions + 6 fill cache functor calls +# sum = 44 + 444 +readAgainProcess.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(27), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 44, 44, 444, 444), + expectedSum = cms.int32(488) +) + +# transitions = 12 events + 9 access input ProcessBlock transitions + 3 fill cache functor calls +# sum = 44 + 444 +readAgainProcess.readProcessBlocksOneAnalyzer3 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644), + expectedSum = cms.int32(488) +) + +# transitions = 12 events + 9 access input ProcessBlock transitions + 3 fill cache functor calls +# sum = 44 + 444 +# filler sum = 3 x 44000 +readAgainProcess.readProcessBlocksOneAnalyzer4 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockT", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + # The expectedByRun test cannot work because the data is from an earlier SubProcess + expectedByRun = cms.vint32(), + expectedFillerSum = cms.untracked.int32(132000), + expectedSum = cms.int32(488) +) + +# transitions = 12 events + 9 access input ProcessBlock transitions + 3 fill cache functor calls +# sum = 44 + 444 +# filler sum = 3 x 55000 +readAgainProcess.readProcessBlocksOneAnalyzer5 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockR", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + # The expectedByRun test cannot work because the data is from an earlier SubProcess + expectedByRun = cms.vint32(), + expectedFillerSum = cms.untracked.int32(165000), + expectedSum = cms.int32(488), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlockT") +) + +readAgainProcess.intProducerBeginProcessBlockRA = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(100000)) + +readAgainProcess.intProducerEndProcessBlockRA = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(1000000)) + +readAgainProcess.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockSubProcessReadAgain.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readAgainProcess.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST', 'READ', 'READAGAIN'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST'), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedAddedProcesses = cms.untracked.vstring('TEST', 'READ', 'READAGAIN'), + expectedTopAddedProcesses = cms.untracked.vstring('TEST'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(10), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedOuterOffset = cms.untracked.vuint32(0), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +readAgainProcess.p = cms.Path( + readAgainProcess.intProducerBeginProcessBlockRA * + readAgainProcess.intProducerEndProcessBlockRA * + readAgainProcess.readProcessBlocksOneAnalyzer1 * + readAgainProcess.readProcessBlocksOneAnalyzer2 * + readAgainProcess.readProcessBlocksOneAnalyzer3 * + readAgainProcess.readProcessBlocksOneAnalyzer4 * + readAgainProcess.readProcessBlocksOneAnalyzer5 +) + +readAgainProcess.e = cms.EndPath( + readAgainProcess.out * + readAgainProcess.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockTEST_cfg.py b/FWCore/Integration/test/testProcessBlockTEST_cfg.py new file mode 100644 index 0000000000000..d823763155127 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockTEST_cfg.py @@ -0,0 +1,81 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root' + ) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockTest.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.eventIntProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.nonEventIntProducer = cms.EDProducer("NonEventIntProducer", + ivalue = cms.int32(1) +) + +process.p = cms.Path( + process.eventIntProducer * + process.transientIntProducerEndProcessBlock * + process.nonEventIntProducer +) + +process.e = cms.EndPath( + process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py b/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py new file mode 100644 index 0000000000000..5a82284617597 --- /dev/null +++ b/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py @@ -0,0 +1,70 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("THREETEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockMergeOfMergedFiles.root', + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(71), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(24193) +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(53), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedSum = cms.int32(1020) +) + +process.out = cms.OutputModule("PoolOutputModule", + fileName = cms.untracked.string('testProcessBlockThreeFileInput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 14, 16, 11, 14, 16, 12, 15, 16, 13, 15, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 4), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(4, 2, 1), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 7), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWCore/Integration/test/testProcessBlock_cfg.py b/FWCore/Integration/test/testProcessBlock_cfg.py deleted file mode 100644 index 9080d38492849..0000000000000 --- a/FWCore/Integration/test/testProcessBlock_cfg.py +++ /dev/null @@ -1,14 +0,0 @@ -import FWCore.ParameterSet.Config as cms - -process = cms.Process("TEST") - -process.source = cms.Source("EmptySource") - -process.maxEvents = cms.untracked.PSet( - input = cms.untracked.int32(1) -) - -process.thingProducer = cms.EDProducer("ThingProducer") - -process.path = cms.Path(process.thingProducer) - diff --git a/FWCore/Integration/test/unit_test_outputs/testConsumesInfo_1.log b/FWCore/Integration/test/unit_test_outputs/testConsumesInfo_1.log index 0f4d964805dcc..ff44dde81f149 100644 --- a/FWCore/Integration/test/unit_test_outputs/testConsumesInfo_1.log +++ b/FWCore/Integration/test/unit_test_outputs/testConsumesInfo_1.log @@ -37,6 +37,7 @@ modules on end path e: modules on end path p1ep2: All modules and modules in the current process whose products they consume: (This does not include modules from previous processes or the source) +(Exclusively considers Event products, not Run, Lumi, or ProcessBlock products) IntProducer/'intProducerA' IntProducer/'intProducerF' IntProducer/'intProducerG' @@ -158,6 +159,7 @@ modules on path path1: intProducerD All modules and modules in the current process whose products they consume: (This does not include modules from previous processes or the source) +(Exclusively considers Event products, not Run, Lumi, or ProcessBlock products) IntProducer/'intProducerF' IntProducer/'intProducerH' IntProducer/'intProducerB' @@ -186,6 +188,7 @@ paths: end paths: ep1 ep2 + endPath modules on path p3: intVectorProducer test @@ -233,8 +236,11 @@ modules on end path ep1: intVectorProducer testManyConsumingProducer modules on end path ep2: +modules on end path endPath: + testOneOutput All modules and modules in the current process whose products they consume: (This does not include modules from previous processes or the source) +(Exclusively considers Event products, not Run, Lumi, or ProcessBlock products) IntProducer/'intProducerF' IntProducer/'intProducerG' IntVectorProducer/'intVectorProducer' @@ -293,6 +299,15 @@ All modules and modules in the current process whose products they consume: TestFindProduct/'processBlockTest1' ConsumingIntProducer/'testManyConsumingProducer' consumes products from these modules: TriggerResultInserter/'TriggerResults' + TestOneOutput/'testOneOutput' consumes products from these modules: + TriggerResultInserter/'TriggerResults' + IntProducer/'intProducerB' + IntProducer/'intProducerC' + IntProducer/'intProducerF' + IntProducer/'intProducerG' + ConsumingIntProducer/'testManyConsumingProducer' + IntProducer/'testStreamingProducer' + IntVectorProducer/'intVectorProducer' TriggerResultInserter/'TriggerResults' PathStatusInserter/'p3' PathStatusInserter/'path1' @@ -300,6 +315,7 @@ All modules and modules in the current process whose products they consume: PathStatusInserter/'path3' EndPathStatusInserter/'ep1' EndPathStatusInserter/'ep2' + EndPathStatusInserter/'endPath' All modules (listed by class and label) and all their consumed products. Consumed products are listed by type, label, instance, process. For products not in the event, 'processBlock', 'run' or 'lumi' is added to indicate the TTree they are from. @@ -410,18 +426,72 @@ For products only read from previous processes, 'skip current process' is added IntProducerEndProcessBlock/'intProducerEndProcessBlock' TestFindProduct/'processBlockTest1' consumes: edmtest::IntProduct 'intProducerBeginProcessBlock' '' '', processBlock - edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'PROD1', processBlock edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'COPY', processBlock - edmtest::IntProduct 'intProducerBeginProcessBlock' '' '', processBlock edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'PROD1', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' '' 'PROD1', processBlock + edmtest::IntProduct 'intProducerBeginProcessBlock' '' '', processBlock edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'COPY', processBlock edmtest::IntProduct 'intProducerEndProcessBlock' '' '', processBlock - edmtest::IntProduct 'intProducerEndProcessBlock' '' 'PROD1', processBlock edmtest::IntProduct 'intProducerEndProcessBlock' '' 'COPY', processBlock edmtest::IntProduct 'intProducerEndProcessBlock' '' 'COPY', processBlock ConsumingIntProducer/'testManyConsumingProducer' consumes: edm::TriggerResults 'TriggerResults' '' '' edm::TriggerResults '' '' '' + TestOneOutput/'testOneOutput' consumes: + edm::TriggerResults 'TriggerResults' '' 'COPY' + edm::TriggerResults 'TriggerResults' '' 'PROD1' + edm::TriggerResults 'TriggerResults' '' 'PROD2' + edmtest::IntProduct 'aliasForInt' '' 'PROD1' + edmtest::IntProduct 'intProducer' '' 'PROD1' + edmtest::IntProduct 'intProducerB' '' 'COPY' + edmtest::IntProduct 'intProducerB' '' 'PROD1' + edmtest::IntProduct 'intProducerB' '' 'PROD2' + edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'COPY', processBlock + edmtest::IntProduct 'intProducerBeginProcessBlock' '' 'PROD1', processBlock + edmtest::IntProduct 'intProducerC' '' 'COPY' + edmtest::IntProduct 'intProducerC' '' 'PROD1' + edmtest::IntProduct 'intProducerD' '' 'PROD1' + edmtest::IntProduct 'intProducerD' '' 'PROD2' + edmtest::IntProduct 'intProducerE' '' 'PROD1' + edmtest::IntProduct 'intProducerEndProcessBlock' '' 'COPY', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' '' 'PROD1', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'four' 'COPY', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'four' 'PROD1', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'three' 'COPY', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'three' 'PROD1', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'two' 'COPY', processBlock + edmtest::IntProduct 'intProducerEndProcessBlock' 'two' 'PROD1', processBlock + edmtest::IntProduct 'intProducerF' '' 'COPY' + edmtest::IntProduct 'intProducerF' '' 'PROD1' + edmtest::IntProduct 'intProducerF' '' 'PROD2' + edmtest::IntProduct 'intProducerG' '' 'COPY' + edmtest::IntProduct 'intProducerG' '' 'PROD1' + edmtest::IntProduct 'intProducerH' '' 'PROD1' + edmtest::IntProduct 'intProducerH' '' 'PROD2' + edmtest::IntProduct 'intProducerI' '' 'PROD1' + edmtest::IntProduct 'intProducerU' '' 'PROD1' + edmtest::IntProduct 'source' '' 'PROD1' + edmtest::IntProduct 'testManyConsumingProducer' '' 'COPY' + edmtest::IntProduct 'testManyConsumingProducer' '' 'PROD1' + edmtest::IntProduct 'testStreamingProducer' '' 'COPY' + edmtest::IntProduct 'testStreamingProducer' '' 'PROD1' + edmtest::Thing 'thingWithMergeProducer' 'beginLumi' 'COPY', lumi + edmtest::Thing 'thingWithMergeProducer' 'beginRun' 'COPY', run + edmtest::Thing 'thingWithMergeProducer' 'endLumi' 'COPY', lumi + edmtest::Thing 'thingWithMergeProducer' 'endRun' 'COPY', run + edmtest::Thing 'thingWithMergeProducer' 'event' 'COPY' + edmtest::ThingWithIsEqual 'thingWithMergeProducer' 'beginLumi' 'COPY', lumi + edmtest::ThingWithIsEqual 'thingWithMergeProducer' 'beginRun' 'COPY', run + edmtest::ThingWithIsEqual 'thingWithMergeProducer' 'endLumi' 'COPY', lumi + edmtest::ThingWithIsEqual 'thingWithMergeProducer' 'endRun' 'COPY', run + edmtest::ThingWithIsEqual 'thingWithMergeProducer' 'event' 'COPY' + edmtest::ThingWithMerge 'thingWithMergeProducer' 'beginLumi' 'COPY', lumi + edmtest::ThingWithMerge 'thingWithMergeProducer' 'beginRun' 'COPY', run + edmtest::ThingWithMerge 'thingWithMergeProducer' 'endLumi' 'COPY', lumi + edmtest::ThingWithMerge 'thingWithMergeProducer' 'endRun' 'COPY', run + edmtest::ThingWithMerge 'thingWithMergeProducer' 'event' 'COPY' + std::vector 'intVectorProducer' '' 'COPY' + std::vector 'intVectorProducer' '' 'PROD1' TriggerResultInserter/'TriggerResults' PathStatusInserter/'p3' PathStatusInserter/'path1' @@ -429,6 +499,7 @@ For products only read from previous processes, 'skip current process' is added PathStatusInserter/'path3' EndPathStatusInserter/'ep1' EndPathStatusInserter/'ep2' + EndPathStatusInserter/'endPath' TestFindProduct sum = 12 TestFindProduct sum = 300 @@ -464,4 +535,4 @@ TestFindProduct sum = 9015 TestFindProduct sum = 0 TestFindProduct sum = 0 TestFindProduct sum = 45057 -TestFindProduct sum = 460034 +TestFindProduct sum = 450034 diff --git a/FWCore/Integration/test/unit_test_outputs/testGetBy1.log b/FWCore/Integration/test/unit_test_outputs/testGetBy1.log index b5d83a975d8bf..4808bdbb018ca 100644 --- a/FWCore/Integration/test/unit_test_outputs/testGetBy1.log +++ b/FWCore/Integration/test/unit_test_outputs/testGetBy1.log @@ -1430,6 +1430,34 @@ GlobalContext: transition = EndProcessBlock runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 ProcessContext: PROD1 be16549dc0c1f4b03231a8b98d235dac +++++ starting: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: COPY 5ea2a17b2b2eaa97af73c630882cd994 + parent ProcessContext: PROD1 be16549dc0c1f4b03231a8b98d235dac + +++++ finished: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: COPY 5ea2a17b2b2eaa97af73c630882cd994 + parent ProcessContext: PROD1 be16549dc0c1f4b03231a8b98d235dac + +++++ starting: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: COPY 5ea2a17b2b2eaa97af73c630882cd994 + parent ProcessContext: PROD1 be16549dc0c1f4b03231a8b98d235dac + +++++ finished: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: COPY 5ea2a17b2b2eaa97af73c630882cd994 + parent ProcessContext: PROD1 be16549dc0c1f4b03231a8b98d235dac + ++++ starting: end process block GlobalContext: transition = EndProcessBlock run: 0 luminosityBlock: 0 diff --git a/FWCore/Integration/test/unit_test_outputs/testGetBy2.log b/FWCore/Integration/test/unit_test_outputs/testGetBy2.log index 20a92494c0884..bf6294e8a33f2 100644 --- a/FWCore/Integration/test/unit_test_outputs/testGetBy2.log +++ b/FWCore/Integration/test/unit_test_outputs/testGetBy2.log @@ -87,6 +87,34 @@ GlobalContext: transition = BeginProcessBlock runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 ProcessContext: PROD2 7da3661f4f7dead5e42f07cf3ddf5a59 +++++ starting: source process block +++++ finished: source process block PROD1 +++++ starting: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 7da3661f4f7dead5e42f07cf3ddf5a59 + +++++ finished: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 7da3661f4f7dead5e42f07cf3ddf5a59 + +++++ starting: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 7da3661f4f7dead5e42f07cf3ddf5a59 + +++++++ starting: write process block for module: label = 'out' id = 5 +++++++ finished: write process block for module: label = 'out' id = 5 +++++ finished: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 7da3661f4f7dead5e42f07cf3ddf5a59 + ++++ starting: source run ++++ finished: source run ++++ starting: global begin run 1 : time = 1 diff --git a/FWCore/ServiceRegistry/interface/ActivityRegistry.h b/FWCore/ServiceRegistry/interface/ActivityRegistry.h index 95428bf3667d9..3b1d2fccce7cf 100644 --- a/FWCore/ServiceRegistry/interface/ActivityRegistry.h +++ b/FWCore/ServiceRegistry/interface/ActivityRegistry.h @@ -214,6 +214,22 @@ namespace edm { void watchPostSourceRun(PostSourceRun::slot_type const& iSlot) { postSourceRunSignal_.connect_front(iSlot); } AR_WATCH_USING_METHOD_1(watchPostSourceRun) + /// signal is emitted before the source starts creating a ProcessBlock + typedef signalslot::Signal PreSourceProcessBlock; + PreSourceProcessBlock preSourceProcessBlockSignal_; + void watchPreSourceProcessBlock(PreSourceProcessBlock::slot_type const& iSlot) { + preSourceProcessBlockSignal_.connect(iSlot); + } + AR_WATCH_USING_METHOD_0(watchPreSourceProcessBlock) + + /// signal is emitted after the source starts creating a ProcessBlock + typedef signalslot::Signal PostSourceProcessBlock; + PostSourceProcessBlock postSourceProcessBlockSignal_; + void watchPostSourceProcessBlock(PostSourceProcessBlock::slot_type const& iSlot) { + postSourceProcessBlockSignal_.connect_front(iSlot); + } + AR_WATCH_USING_METHOD_1(watchPostSourceProcessBlock) + /// signal is emitted before the source opens a file typedef signalslot::Signal PreOpenFile; PreOpenFile preOpenFileSignal_; diff --git a/FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h b/FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h new file mode 100644 index 0000000000000..a847e9217d981 --- /dev/null +++ b/FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h @@ -0,0 +1,8 @@ +#ifndef Framework_ServiceRegistry_fwd_h +#define Framework_ServiceRegistry_fwd_h + +namespace edm { + class ModuleCallingContext; + class StreamContext; +} // namespace edm +#endif diff --git a/FWCore/ServiceRegistry/src/ActivityRegistry.cc b/FWCore/ServiceRegistry/src/ActivityRegistry.cc index 865a2cfc1918a..7dc66bae65a4f 100644 --- a/FWCore/ServiceRegistry/src/ActivityRegistry.cc +++ b/FWCore/ServiceRegistry/src/ActivityRegistry.cc @@ -105,6 +105,9 @@ namespace edm { preSourceRunSignal_.connect(std::cref(iOther.preSourceRunSignal_)); postSourceRunSignal_.connect(std::cref(iOther.postSourceRunSignal_)); + preSourceProcessBlockSignal_.connect(std::cref(iOther.preSourceProcessBlockSignal_)); + postSourceProcessBlockSignal_.connect(std::cref(iOther.postSourceProcessBlockSignal_)); + preOpenFileSignal_.connect(std::cref(iOther.preOpenFileSignal_)); postOpenFileSignal_.connect(std::cref(iOther.postOpenFileSignal_)); @@ -325,6 +328,9 @@ namespace edm { copySlotsToFrom(preSourceRunSignal_, iOther.preSourceRunSignal_); copySlotsToFromReverse(postSourceRunSignal_, iOther.postSourceRunSignal_); + copySlotsToFrom(preSourceProcessBlockSignal_, iOther.preSourceProcessBlockSignal_); + copySlotsToFromReverse(postSourceProcessBlockSignal_, iOther.postSourceProcessBlockSignal_); + copySlotsToFrom(preOpenFileSignal_, iOther.preOpenFileSignal_); copySlotsToFromReverse(postOpenFileSignal_, iOther.postOpenFileSignal_); diff --git a/FWCore/Services/plugins/Tracer.cc b/FWCore/Services/plugins/Tracer.cc index 2e8fa7a2381e4..636109fd46d0d 100644 --- a/FWCore/Services/plugins/Tracer.cc +++ b/FWCore/Services/plugins/Tracer.cc @@ -85,6 +85,9 @@ namespace edm { void preSourceRun(RunIndex); void postSourceRun(RunIndex); + void preSourceProcessBlock(); + void postSourceProcessBlock(std::string const&); + void preOpenFile(std::string const&, bool); void postOpenFile(std::string const&, bool); @@ -268,6 +271,9 @@ Tracer::Tracer(ParameterSet const& iPS, ActivityRegistry& iRegistry) iRegistry.watchPreSourceRun(this, &Tracer::preSourceRun); iRegistry.watchPostSourceRun(this, &Tracer::postSourceRun); + iRegistry.watchPreSourceProcessBlock(this, &Tracer::preSourceProcessBlock); + iRegistry.watchPostSourceProcessBlock(this, &Tracer::postSourceProcessBlock); + iRegistry.watchPreOpenFile(this, &Tracer::preOpenFile); iRegistry.watchPostOpenFile(this, &Tracer::postOpenFile); @@ -497,6 +503,7 @@ void Tracer::preBeginJob(PathsAndConsumesOfModulesBase const& pathsAndConsumes, std::vector const& allModules = pathsAndConsumes.allModules(); out << "All modules and modules in the current process whose products they consume:\n"; out << "(This does not include modules from previous processes or the source)\n"; + out << "(Exclusively considers Event products, not Run, Lumi, or ProcessBlock products)\n"; for (auto const& module : allModules) { out << " " << module->moduleName() << "/\'" << module->moduleLabel() << "\'"; unsigned int moduleID = module->id(); @@ -586,6 +593,16 @@ void Tracer::postSourceRun(RunIndex index) { LogAbsolute("Tracer") << TimeStamper(printTimestamps_) << indention_ << indention_ << " finished: source run"; } +void Tracer::preSourceProcessBlock() { + LogAbsolute("Tracer") << TimeStamper(printTimestamps_) << indention_ << indention_ + << " starting: source process block"; +} + +void Tracer::postSourceProcessBlock(std::string const& processName) { + LogAbsolute("Tracer") << TimeStamper(printTimestamps_) << indention_ << indention_ + << " finished: source process block " << processName; +} + void Tracer::preOpenFile(std::string const& lfn, bool b) { LogAbsolute out("Tracer"); out << TimeStamper(printTimestamps_); diff --git a/FWCore/Sources/interface/RawInputSource.h b/FWCore/Sources/interface/RawInputSource.h index 3504ae6f72fd5..27ef1f5bfce97 100644 --- a/FWCore/Sources/interface/RawInputSource.h +++ b/FWCore/Sources/interface/RawInputSource.h @@ -37,7 +37,7 @@ namespace edm { void rewind_() override; ItemType getNextItemType() override; void closeFile_() final; - std::unique_ptr readFile_() final; + std::shared_ptr readFile_() final; virtual void genuineCloseFile() {} virtual void genuineReadFile() {} diff --git a/FWCore/Sources/src/RawInputSource.cc b/FWCore/Sources/src/RawInputSource.cc index 855d76b11905d..8d246863a0b3a 100644 --- a/FWCore/Sources/src/RawInputSource.cc +++ b/FWCore/Sources/src/RawInputSource.cc @@ -114,12 +114,12 @@ namespace edm { } } - std::unique_ptr RawInputSource::readFile_() { + std::shared_ptr RawInputSource::readFile_() { if (!fakeInputFileTransition_) { genuineReadFile(); } fakeInputFileTransition_ = false; - return std::make_unique(); + return std::make_shared(); } } // namespace edm diff --git a/FWCore/TFWLiteSelector/src/TFWLiteSelectorBasic.cc b/FWCore/TFWLiteSelector/src/TFWLiteSelectorBasic.cc index 01bdada2fa949..563c1853e14c7 100644 --- a/FWCore/TFWLiteSelector/src/TFWLiteSelectorBasic.cc +++ b/FWCore/TFWLiteSelector/src/TFWLiteSelectorBasic.cc @@ -25,6 +25,7 @@ #include "DataFormats/Provenance/interface/EventAuxiliary.h" #include "DataFormats/Provenance/interface/EventEntryDescription.h" // kludge to allow compilation #include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/FileFormatVersion.h" #include "DataFormats/Provenance/interface/LuminosityBlockAuxiliary.h" #include "DataFormats/Provenance/interface/ModuleDescription.h" @@ -322,6 +323,7 @@ Bool_t TFWLiteSelectorBasic::Process(Long64_t iEntry) { branchListIndexBranch->SetAddress(&pBranchListIndexes); branchListIndexBranch->GetEntry(iEntry); m_->branchIDListHelper_->fixBranchListIndexes(branchListIndexes); + edm::EventToProcessBlockIndexes dummyEventToProcessBlockIndexes; try { m_->reader_->setEntry(iEntry); @@ -334,6 +336,7 @@ Bool_t TFWLiteSelectorBasic::Process(Long64_t iEntry) { history, std::move(eventSelectionIDs), std::move(branchListIndexes), + dummyEventToProcessBlockIndexes, *(m_->provRetriever_), m_->reader_.get()); lbp->setRunPrincipal(rp); diff --git a/FWCore/TestProcessor/interface/TestProcessor.h b/FWCore/TestProcessor/interface/TestProcessor.h index 75a46f2476ae7..aa8f11dcf984c 100644 --- a/FWCore/TestProcessor/interface/TestProcessor.h +++ b/FWCore/TestProcessor/interface/TestProcessor.h @@ -27,6 +27,7 @@ #include "tbb/task_group.h" // user include files +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/SharedResourcesAcquirer.h" #include "FWCore/Framework/src/PrincipalCache.h" #include "FWCore/Framework/src/SignallingProductRegistry.h" @@ -331,6 +332,7 @@ This simulates a problem happening early in the job which causes processing not std::shared_ptr actReg_; // We do not use propagate_const because the registry itself is mutable. std::shared_ptr preg_; std::shared_ptr branchIDListHelper_; + std::shared_ptr processBlockHelper_; std::shared_ptr thinnedAssociationsHelper_; ServiceToken serviceToken_; std::unique_ptr espController_; diff --git a/FWCore/TestProcessor/src/TestProcessor.cc b/FWCore/TestProcessor/src/TestProcessor.cc index 4864e53f94df4..34f6c3b913670 100644 --- a/FWCore/TestProcessor/src/TestProcessor.cc +++ b/FWCore/TestProcessor/src/TestProcessor.cc @@ -16,6 +16,7 @@ #include "FWCore/TestProcessor/interface/TestProcessor.h" #include "FWCore/TestProcessor/interface/EventSetupTestHelper.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" #include "FWCore/Framework/interface/ScheduleItems.h" #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" @@ -161,7 +162,9 @@ namespace edm { preg_->addProduct(product); } - schedule_ = items.initSchedule(*psetPtr, false, preallocations_, &processContext_); + processBlockHelper_ = std::make_shared(); + + schedule_ = items.initSchedule(*psetPtr, false, preallocations_, &processContext_, *processBlockHelper_); // set the data members act_table_ = std::move(items.act_table_); actReg_ = items.actReg_; @@ -410,7 +413,7 @@ namespace edm { espController_->finishConfiguration(); - schedule_->beginJob(*preg_, esp_->recordsToProxyIndices()); + schedule_->beginJob(*preg_, esp_->recordsToProxyIndices(), *processBlockHelper_); actReg_->postBeginJobSignal_(); for (unsigned int i = 0; i < preallocations_.numberOfStreams(); ++i) { diff --git a/FWCore/Utilities/interface/FWCoreUtiliesFwd.h b/FWCore/Utilities/interface/FWCoreUtiliesFwd.h new file mode 100644 index 0000000000000..9262d940750c3 --- /dev/null +++ b/FWCore/Utilities/interface/FWCoreUtiliesFwd.h @@ -0,0 +1,9 @@ +#ifndef FWCore_Utilities_FWCoreUtilitiesFwd_h +#define FWCore_Utilities_FWCoreUtilitiesFwd_h + +namespace edm { + class TypeID; + struct ProductLabels; +} // namespace edm + +#endif diff --git a/IOPool/Input/src/PoolSource.cc b/IOPool/Input/src/PoolSource.cc index 2a3f0fdf12d7c..052b77f184a20 100644 --- a/IOPool/Input/src/PoolSource.cc +++ b/IOPool/Input/src/PoolSource.cc @@ -154,8 +154,8 @@ namespace edm { InputFile::reportReadBranches(); } - std::unique_ptr PoolSource::readFile_() { - std::unique_ptr fb = primaryFileSequence_->readFile_(); + std::shared_ptr PoolSource::readFile_() { + std::shared_ptr fb = primaryFileSequence_->readFile_(); if (secondaryFileSequence_) { fb->setNotFastClonable(FileBlock::HasSecondaryFileSequence); } @@ -170,6 +170,16 @@ namespace edm { return primaryFileSequence_->readLuminosityBlockAuxiliary_(); } + void PoolSource::fillProcessBlockHelper_() { primaryFileSequence_->fillProcessBlockHelper_(); } + + bool PoolSource::nextProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + return primaryFileSequence_->nextProcessBlock_(processBlockPrincipal); + } + + void PoolSource::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + primaryFileSequence_->readProcessBlock_(processBlockPrincipal); + } + void PoolSource::readRun_(RunPrincipal& runPrincipal) { primaryFileSequence_->readRun_(runPrincipal); if (secondaryFileSequence_ && !branchIDsToReplace_[InRun].empty()) { diff --git a/IOPool/Input/src/PoolSource.h b/IOPool/Input/src/PoolSource.h index c471abe77c395..ed7deb115f91f 100644 --- a/IOPool/Input/src/PoolSource.h +++ b/IOPool/Input/src/PoolSource.h @@ -57,7 +57,10 @@ namespace edm { private: std::shared_ptr readRunAuxiliary_() override; void readRun_(RunPrincipal& runPrincipal) override; - std::unique_ptr readFile_() override; + void fillProcessBlockHelper_() override; + bool nextProcessBlock_(ProcessBlockPrincipal&) override; + void readProcessBlock_(ProcessBlockPrincipal&) override; + std::shared_ptr readFile_() override; void closeFile_() override; void endJob() override; bool readIt(EventID const& id, EventPrincipal& eventPrincipal, StreamContext& streamContext) override; diff --git a/IOPool/Input/src/RepeatingCachedRootSource.cc b/IOPool/Input/src/RepeatingCachedRootSource.cc index 74bffb010cad1..56774ff90f0d3 100644 --- a/IOPool/Input/src/RepeatingCachedRootSource.cc +++ b/IOPool/Input/src/RepeatingCachedRootSource.cc @@ -17,6 +17,7 @@ #include "DataFormats/Provenance/interface/ProductRegistry.h" #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" #include "DataFormats/Provenance/interface/BranchIDListHelper.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/ProcessConfiguration.h" #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "DataFormats/Common/interface/WrapperBase.h" @@ -270,6 +271,7 @@ std::unique_ptr RepeatingCachedRootSource::makeRootFile( selectorRules_, InputType::Primary, branchIDListHelper(), + nullptr, thinnedAssociationsHelper(), nullptr, // associationsFromSecondary duplicateChecker, @@ -330,6 +332,7 @@ void RepeatingCachedRootSource::readEvent_(EventPrincipal& eventPrincipal) { history, selectionIDs_, branchListIndexes_, + EventToProcessBlockIndexes(), provRetriever_, &delayedReaders_[eventPrincipal.streamID().value()]); } diff --git a/IOPool/Input/src/RootDelayedReader.cc b/IOPool/Input/src/RootDelayedReader.cc index 9d7df83aecc7b..c10eea97cf315 100644 --- a/IOPool/Input/src/RootDelayedReader.cc +++ b/IOPool/Input/src/RootDelayedReader.cc @@ -76,7 +76,7 @@ namespace edm { std::unique_ptr edp = getWrapperBasePtr(p, branchInfo->offsetToWrapperBase_); br->SetAddress(&p); try { - //Run and Lumi only have 1 entry number, which is index 0 + //Run, Lumi, and ProcessBlock only have 1 entry number, which is index 0 tree_.getEntry(br, tree_.entryNumberForIndex(tree_.branchType() == InEvent ? ep->transitionIndex() : 0)); } catch (edm::Exception& exception) { exception.addContext("Rethrowing an exception that happened on a different thread."); diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index 043cf75bef5dd..d44daf63e5fb0 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -19,14 +19,17 @@ #include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" #include "DataFormats/Provenance/interface/ProductRegistry.h" #include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" #include "DataFormats/Provenance/interface/StoredProductProvenance.h" #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "DataFormats/Provenance/interface/RunID.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" #include "FWCore/Framework/interface/FileBlock.h" #include "FWCore/Framework/interface/EventPrincipal.h" #include "FWCore/Framework/interface/ProductSelector.h" #include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" #include "FWCore/Framework/interface/MergeableRunProductMetadata.h" +#include "FWCore/Framework/interface/ProcessBlockPrincipal.h" #include "FWCore/Framework/interface/RunPrincipal.h" #include "FWCore/Framework/interface/SharedResourcesAcquirer.h" #include "FWCore/Framework/src/SharedResourcesRegistry.h" @@ -64,6 +67,7 @@ #include "TTreeCache.h" #include +#include #include namespace edm { @@ -155,6 +159,7 @@ namespace edm { ProductSelectorRules const& productSelectorRules, InputType inputType, std::shared_ptr branchIDListHelper, + ProcessBlockHelper* processBlockHelper, std::shared_ptr thinnedAssociationsHelper, std::vector const* associationsFromSecondary, std::shared_ptr duplicateChecker, @@ -224,12 +229,17 @@ namespace edm { productRegistry_(), branchIDLists_(), branchIDListHelper_(branchIDListHelper), + processBlockHelper_(processBlockHelper), fileThinnedAssociationsHelper_(), thinnedAssociationsHelper_(thinnedAssociationsHelper), processingMode_(processingMode), runHelper_(runHelper), newBranchToOldBranch_(), eventHistoryTree_(nullptr), + eventToProcessBlockIndexesBranch_( + inputType == InputType::Primary + ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str()) + : nullptr), history_(), branchChildren_(new BranchChildren), duplicateChecker_(duplicateChecker), @@ -242,10 +252,9 @@ namespace edm { inputType_(inputType) { hasNewlyDroppedBranch_.fill(false); - treePointers_[InEvent] = &eventTree_; - treePointers_[InLumi] = &lumiTree_; - treePointers_[InRun] = &runTree_; - treePointers_[InProcess] = nullptr; + treePointers_.emplace_back(&eventTree_); + treePointers_.emplace_back(&lumiTree_); + treePointers_.emplace_back(&runTree_); // Read the metadata tree. // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. @@ -276,6 +285,15 @@ namespace edm { metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr); } + storedProcessBlockHelper_ = std::make_unique(); + StoredProcessBlockHelper& storedProcessBlockHelper = *storedProcessBlockHelper_; + StoredProcessBlockHelper* pStoredProcessBlockHelper = storedProcessBlockHelper_.get(); + if (inputType == InputType::Primary) { + if (metaDataTree->FindBranch(poolNames::processBlockHelperBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper); + } + } + StoredMergeableRunProductMetadata* smrc = nullptr; if (inputType == InputType::Primary) { smrc = &*storedMergeableRunProductMetadata_; @@ -290,7 +308,7 @@ namespace edm { ProductRegistry* ppReg = &inputProdDescReg; metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg)); - typedef std::map PsetMap; + using PsetMap = std::map; PsetMap psetMap; PsetMap* psetMapPtr = &psetMap; if (metaDataTree->FindBranch(poolNames::parameterSetMapBranchName().c_str()) != nullptr) { @@ -306,7 +324,7 @@ namespace edm { << "Could not find tree " << poolNames::parameterSetsTreeName() << " in the input file.\n"; } - typedef std::pair IdToBlobs; + using IdToBlobs = std::pair; IdToBlobs idToBlob; IdToBlobs* pIdToBlob = &idToBlob; psetTree->SetBranchAddress(poolNames::idToParameterSetBlobsBranchName().c_str(), &pIdToBlob); @@ -497,17 +515,9 @@ namespace edm { runHelper_->setForcedRunOffset(indexIntoFileBegin_ == indexIntoFileEnd_ ? 1 : indexIntoFileBegin_.run()); eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin(); - // Set product presence information in the product registry. - ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator(); - for (auto& product : pList) { - BranchDescription& prod = product.second; - prod.init(); - if (prod.branchType() == InProcess) { - // ProcessBlock input not implemented yet - continue; - } - treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName())); - } + makeProcessBlockRootTrees(filePtr, treeMaxVirtualSize, enablePrefetching, inputType, storedProcessBlockHelper); + + setPresenceInProductRegistry(inputProdDescReg, storedProcessBlockHelper); auto newReg = std::make_unique(); @@ -531,11 +541,15 @@ namespace edm { newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), prod.branchName())); } } - dropOnInput(*newReg, productSelectorRules, dropDescendants, inputType); + + dropOnInputAndReorder( + *newReg, productSelectorRules, dropDescendants, inputType, storedProcessBlockHelper, processBlockHelper); + if (inputType == InputType::SecondaryFile) { thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_, *associationsFromSecondary); } else if (inputType == InputType::Primary) { + processBlockHelper->initializeFromPrimaryInput(storedProcessBlockHelper); thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_); } @@ -549,6 +563,10 @@ namespace edm { product.second.setOnDemand(true); } + for (auto& processBlockTree : processBlockTrees_) { + treePointers_.push_back(processBlockTree.get()); + } + // freeze the product registry newReg->setFrozen(inputType != InputType::Primary); productRegistry_.reset(newReg.release()); @@ -558,18 +576,22 @@ namespace edm { ProductRegistry::ProductList const& prodList = productRegistry()->productList(); { - std::array nBranches; - nBranches.fill(0); + std::vector nBranches(treePointers_.size(), 0); for (auto const& product : prodList) { - ++nBranches[product.second.branchType()]; + if (product.second.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), product.second.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + ++nBranches[numberOfRunLumiEventProductTrees + index]; + } + } else { + ++nBranches[product.second.branchType()]; + } } int i = 0; - for (auto t : treePointers_) { - if (t == nullptr) { - // ProcessBlock input not implemented yet - continue; - } + for (auto& t : treePointers_) { t->numberOfBranchesToAdd(nBranches[i]); ++i; } @@ -577,10 +599,16 @@ namespace edm { for (auto const& product : prodList) { BranchDescription const& prod = product.second; if (prod.branchType() == InProcess) { - // ProcessBlock input not implemented yet - continue; + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch(prod, + newBranchToOldBranch(prod.branchName())); + } + } else { + treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName())); } - treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName())); } // Determine if this file is fast clonable. @@ -595,6 +623,9 @@ namespace edm { // Train the run and lumi trees. runTree_.trainCache("*"); lumiTree_.trainCache("*"); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTree->trainCache("*"); + } } RootFile::~RootFile() {} @@ -724,14 +755,24 @@ namespace edm { } } - std::unique_ptr RootFile::createFileBlock() const { - return std::make_unique(fileFormatVersion(), + std::shared_ptr RootFile::createFileBlock() { + std::vector processBlockTrees; + std::vector processesWithProcessBlockTrees; + processBlockTrees.reserve(processBlockTrees_.size()); + processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTrees.push_back(processBlockTree->tree()); + processesWithProcessBlockTrees.push_back(processBlockTree->processName()); + } + return std::make_shared(fileFormatVersion(), eventTree_.tree(), eventTree_.metaTree(), lumiTree_.tree(), lumiTree_.metaTree(), runTree_.tree(), runTree_.metaTree(), + std::move(processBlockTrees), + std::move(processesWithProcessBlockTrees), whyNotFastClonable(), hasNewlyDroppedBranch(), file_, @@ -740,6 +781,25 @@ namespace edm { branchChildren()); } + void RootFile::updateFileBlock(FileBlock& fileBlock) { + std::vector processBlockTrees; + std::vector processesWithProcessBlockTrees; + processBlockTrees.reserve(processBlockTrees_.size()); + processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTrees.push_back(processBlockTree->tree()); + processesWithProcessBlockTrees.push_back(processBlockTree->processName()); + } + fileBlock.updateTTreePointers(eventTree_.tree(), + eventTree_.metaTree(), + lumiTree_.tree(), + lumiTree_.metaTree(), + runTree_.tree(), + runTree_.metaTree(), + std::move(processBlockTrees), + std::move(processesWithProcessBlockTrees)); + } + std::string const& RootFile::newBranchToOldBranch(std::string const& newBranch) const { std::map::const_iterator it = newBranchToOldBranch_.find(newBranch); if (it != newBranchToOldBranch_.end()) { @@ -1216,10 +1276,6 @@ namespace edm { // Just to play it safe, zero all pointers to objects in the InputFile to be closed. eventHistoryTree_ = nullptr; for (auto& treePointer : treePointers_) { - if (treePointer == nullptr) { - // ProcessBlock input not implemented yet - continue; - } treePointer->close(); treePointer = nullptr; } @@ -1251,6 +1307,26 @@ namespace edm { return fillThisEventAuxiliary(); } + void RootFile::fillEventToProcessBlockIndexes() { + TBranch* eventToProcessBlockIndexesBranch = get_underlying_safe(eventToProcessBlockIndexesBranch_); + if (eventToProcessBlockIndexesBranch == nullptr) { + if (processBlockHelper_.get() == nullptr) { + eventToProcessBlockIndexes_.setIndex(0); + } else { + eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset()); + } + } else { + if (processBlockHelper_->cacheIndexVectorsPerFile().back() == 1u) { + eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset()); + } else { + EventToProcessBlockIndexes* pEventToProcessBlockIndexes = &eventToProcessBlockIndexes_; + eventTree_.fillBranchEntry(eventToProcessBlockIndexesBranch, pEventToProcessBlockIndexes); + unsigned int updatedIndex = eventToProcessBlockIndexes_.index() + processBlockHelper_->outerOffset(); + eventToProcessBlockIndexes_.setIndex(updatedIndex); + } + } + } + bool RootFile::fillEventHistory(EventAuxiliary& evtAux, EventSelectionIDVector& eventSelectionIDs, BranchListIndexes& branchListIndexes, @@ -1470,6 +1546,7 @@ namespace edm { const_cast(evtAux.id()).setLuminosityBlockNumber(evtAux.oldLuminosityBlock()); evtAux.resetObsoleteInfo(); } + fillEventToProcessBlockIndexes(); EventSelectionIDVector eventSelectionIDs; BranchListIndexes branchListIndexes; if (!fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes, assertOnFailure)) { @@ -1484,6 +1561,7 @@ namespace edm { history, std::move(eventSelectionIDs), std::move(branchListIndexes), + eventToProcessBlockIndexes_, *(makeProductProvenanceRetriever(principal.streamID().value())), eventTree_.resetAndGetRootDelayedReader()); @@ -1568,6 +1646,65 @@ namespace edm { return runAuxiliary; } + void RootFile::fillProcessBlockHelper_() { + assert(inputType_ == InputType::Primary); + std::vector nEntries; + nEntries.reserve(processBlockTrees_.size()); + for (auto const& processBlockTree : processBlockTrees_) { + nEntries.push_back(processBlockTree->entries()); + } + processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, std::move(nEntries)); + storedProcessBlockHelper_ = + std::make_unique(); // propagate_const has no reset() function + } + + bool RootFile::initializeFirstProcessBlockEntry() { + if (processBlockTrees_[currentProcessBlockTree_]->entryNumber() == IndexIntoFile::invalidEntry) { + processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0); + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + return true; + } + return false; + } + + bool RootFile::endOfProcessBlocksReached() const { return currentProcessBlockTree_ >= processBlockTrees_.size(); } + + bool RootFile::nextProcessBlock_(ProcessBlockPrincipal&) { + assert(inputType_ == InputType::Primary); + if (endOfProcessBlocksReached()) { + return false; + } + if (initializeFirstProcessBlockEntry()) { + return true; + } + // With the current design, the RootFile should always be + // set to a valid ProcessBlock entry in one of the TTrees + // if it not at the end. + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + // Try for next entry in the same TTree + if (processBlockTrees_[currentProcessBlockTree_]->nextWithCache()) { + return true; + } + // Next ProcessBlock TTree + ++currentProcessBlockTree_; + if (endOfProcessBlocksReached()) { + return false; + } + // With current design there should always be at least one entry. + // Initialize for that entry. + processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0); + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + return true; + } + + void RootFile::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(inputType_ == InputType::Primary); + RootTree* rootTree = processBlockTrees_[currentProcessBlockTree_].get(); + rootTree->insertEntryForIndex(0); + assert(!rootTree->processName().empty()); + processBlockPrincipal.fillProcessBlockPrincipal(rootTree->processName(), rootTree->resetAndGetRootDelayedReader()); + } + void RootFile::readRun_(RunPrincipal& runPrincipal) { MergeableRunProductMetadata* mergeableRunProductMetadata = nullptr; if (inputType_ == InputType::Primary) { @@ -1717,6 +1854,33 @@ namespace edm { } } + void RootFile::setPresenceInProductRegistry(ProductRegistry& inputProdDescReg, + StoredProcessBlockHelper const& storedProcessBlockHelper) { + // Set product presence information in the product registry. + // "Presence" is a boolean that is true if and only if the TBranch exists + // in the TTree (except it will be false for ProcessBlock products in non-Primary + // input files). + ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator(); + for (auto& product : pList) { + BranchDescription& prod = product.second; + prod.init(); + if (prod.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + processBlockTrees_[index]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + } else { + // Given current rules for saving BranchDescriptions, this case should only occur + // in non-Primary sequences. + prod.setDropped(true); + } + } else { + treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + } + } + } + void RootFile::markBranchToBeDropped(bool dropDescendants, BranchDescription const& branch, std::set& branchesToDrop, @@ -1728,19 +1892,16 @@ namespace edm { } } - void RootFile::dropOnInput(ProductRegistry& reg, - ProductSelectorRules const& rules, - bool dropDescendants, - InputType inputType) { - // This is the selector for drop on input. - ProductSelector productSelector; - productSelector.initialize(rules, reg.allBranchDescriptions()); - - std::vector associationDescriptions; - + void RootFile::dropOnInputAndReorder(ProductRegistry& reg, + ProductSelectorRules const& rules, + bool dropDescendants, + InputType inputType, + StoredProcessBlockHelper& storedProcessBlockHelper, + ProcessBlockHelper const* processBlockHelper) { ProductRegistry::ProductList& prodList = reg.productListUpdator(); - // Do drop on input. On the first pass, just fill in a set of branches to be dropped. - std::set branchesToDrop; + + // First fill in a map we will need to navigate to descendants + // in the case of EDAliases. std::map droppedToKeptAlias; for (auto const& product : prodList) { BranchDescription const& prod = product.second; @@ -1748,10 +1909,22 @@ namespace edm { droppedToKeptAlias[prod.originalBranchID()] = prod.branchID(); } } + + // This object will select products based on the branchName and the + // keep and drop statements which are in the source configuration. + ProductSelector productSelector; + productSelector.initialize(rules, reg.allBranchDescriptions()); + + // In this pass, fill in a set of branches to be dropped. + // Don't drop anything yet. + std::set branchesToDrop; + std::vector associationDescriptions; for (auto const& product : prodList) { BranchDescription const& prod = product.second; - // Special handling for ThinnedAssociations - if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) { + if (inputType != InputType::Primary && prod.branchType() == InProcess) { + markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias); + } else if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) { + // Special handling for ThinnedAssociations if (inputType != InputType::SecondarySource) { associationDescriptions.push_back(&prod); } else { @@ -1803,19 +1976,26 @@ namespace edm { } // On this pass, actually drop the branches. + std::set processesWithKeptProcessBlockProducts; std::set::const_iterator branchesToDropEnd = branchesToDrop.end(); for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) { BranchDescription const& prod = it->second; bool drop = branchesToDrop.find(prod.branchID()) != branchesToDropEnd; if (drop) { if (!prod.dropped()) { - if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation)) { + if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation) && + prod.branchType() != InProcess) { LogWarning("RootFile") << "Branch '" << prod.branchName() << "' is being dropped from the input\n" << "of file '" << file_ << "' because it is dependent on a branch\n" << "that was explicitly dropped.\n"; } - // ProcessBlock input is not implemented yet - if (prod.branchType() != InProcess) { + if (prod.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + assert(it != processes.end()); + auto index = std::distance(processes.begin(), it); + processBlockTrees_[index]->dropBranch(newBranchToOldBranch(prod.branchName())); + } else { treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName())); } hasNewlyDroppedBranch_[prod.branchType()] = true; @@ -1824,10 +2004,15 @@ namespace edm { ++it; prodList.erase(icopy); } else { + if (prod.branchType() == InProcess && prod.present()) { + processesWithKeptProcessBlockProducts.insert(prod.processName()); + } ++it; } } + dropProcessesAndReorder(storedProcessBlockHelper, processesWithKeptProcessBlockProducts, processBlockHelper); + // Drop on input mergeable run and lumi products, this needs to be invoked for secondary file input if (inputType == InputType::SecondaryFile) { TString tString; @@ -1852,12 +2037,96 @@ namespace edm { } } + void RootFile::dropProcessesAndReorder(StoredProcessBlockHelper& storedProcessBlockHelper, + std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelper const* processBlockHelper) { + // Modify storedProcessBlockHelper and processBlockTrees_ + // This should account for dropOnInput and also make the + // order of process blocks in input files after the first + // be the same as the first. Processes with no ProcessBlock + // products should be removed. After this executes, + // the items in storedProcessBlockHelper + // and processBlockTrees should be in exact one to one + // correspondence and in the same order. For input files + // after the first, these items should be either the same + // as or a subset of the items in processBlockHelper and in + // the same order. + + if (processBlockTrees_.empty()) { + return; + } + + std::vector nEntries; + nEntries.reserve(processBlockTrees_.size()); + for (auto const& processBlockTree : processBlockTrees_) { + nEntries.push_back(processBlockTree->entries()); + } + + bool firstInputFile = !processBlockHelper->initializedFromInput(); + bool isModified = false; + std::vector finalIndexToStoredIndex; + + if (firstInputFile) { + isModified = processBlockHelper->firstFileDropProcessesAndReorderStored( + storedProcessBlockHelper.processesWithProcessBlockProducts(), + storedProcessBlockHelper.processBlockCacheIndices(), + processesWithKeptProcessBlockProducts, + nEntries, + finalIndexToStoredIndex); + } else { + isModified = processBlockHelper->dropProcessesAndReorderStored( + storedProcessBlockHelper.processesWithProcessBlockProducts(), + storedProcessBlockHelper.processBlockCacheIndices(), + processesWithKeptProcessBlockProducts, + nEntries, + finalIndexToStoredIndex, + processBlockHelper->processesWithProcessBlockProducts()); + } + + // At this point, any modifications to storedProcessBlockHelper are done. + // Make consistent changes to processBlockTrees_ and this will cause + // unneeded RootTrees to be deleted. + if (isModified) { + std::vector>> newProcessBlockTrees; + unsigned int nFinalProducts = storedProcessBlockHelper.processesWithProcessBlockProducts().size(); + for (unsigned int j = 0; j < nFinalProducts; ++j) { + unsigned int iStored = finalIndexToStoredIndex[j]; + newProcessBlockTrees.push_back(std::move(processBlockTrees_[iStored])); + } + processBlockTrees_.swap(newProcessBlockTrees); + } + } + void RootFile::setSignals( signalslot::Signal const* preEventReadSource, signalslot::Signal const* postEventReadSource) { eventTree_.setSignals(preEventReadSource, postEventReadSource); } + void RootFile::makeProcessBlockRootTrees(std::shared_ptr filePtr, + int treeMaxVirtualSize, + bool enablePrefetching, + InputType inputType, + StoredProcessBlockHelper const& storedProcessBlockHelper) { + // When this functions returns there will be exactly a 1-to-1 correspondence between the + // processes listed in storedProcessBlockHelper and the RootTree objects created. processBlockTrees_ + // has pointers to the RootTree's and will be filled in the same order. The RootTree constructor + // will throw an exception if one of these TTree's is not in the file and this should be all of + // the ProcessBlock TTree's in the file. (later in the RootFile constructor, dropOnInput might + // remove some and also reordering may occur). + for (auto const& process : storedProcessBlockHelper.processesWithProcessBlockProducts()) { + processBlockTrees_.emplace_back(std::make_unique(filePtr, + InProcess, + process, + 1, + treeMaxVirtualSize, + roottree::defaultNonEventCacheSize, + roottree::defaultNonEventLearningEntries, + enablePrefetching, + inputType)); + } + } + std::unique_ptr RootFile::makeProvenanceReaderMaker(InputType inputType) { if (fileFormatVersion_.storedProductProvenanceUsed()) { readParentageTree(inputType); diff --git a/IOPool/Input/src/RootFile.h b/IOPool/Input/src/RootFile.h index 19f9b87bec6a9..13e20f81e15bd 100644 --- a/IOPool/Input/src/RootFile.h +++ b/IOPool/Input/src/RootFile.h @@ -16,18 +16,25 @@ RootFile.h // used by ROOT input sources #include "DataFormats/Provenance/interface/EventEntryDescription.h" // backward compatibility #include "DataFormats/Provenance/interface/EventProcessHistoryID.h" // backward compatibility #include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/FileFormatVersion.h" #include "DataFormats/Provenance/interface/FileID.h" #include "DataFormats/Provenance/interface/History.h" #include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputSource.h" #include "FWCore/Utilities/interface/InputType.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" + +#include "TBranch.h" #include #include #include +#include #include #include #include @@ -52,7 +59,7 @@ namespace edm { class RunHelperBase; class ThinnedAssociationsHelper; - typedef std::map EntryDescriptionMap; + using EntryDescriptionMap = std::map; class MakeProvenanceReader { public: @@ -63,7 +70,7 @@ namespace edm { class RootFile { public: - typedef std::array RootTreePtrArray; + // Constructor used by RootPrimaryFileSequence RootFile(std::string const& fileName, ProcessConfiguration const& processConfiguration, std::string const& logicalFileName, @@ -81,6 +88,7 @@ namespace edm { ProductSelectorRules const& productSelectorRules, InputType inputType, std::shared_ptr branchIDListHelper, + ProcessBlockHelper*, std::shared_ptr thinnedAssociationsHelper, std::vector const* associationsFromSecondary, std::shared_ptr duplicateChecker, @@ -95,6 +103,7 @@ namespace edm { bool enablePrefetching, bool enforceGUIDInFileName); + // Constructor used by RootSecondaryFileSequence RootFile(std::string const& fileName, ProcessConfiguration const& processConfiguration, std::string const& logicalFileName, @@ -134,6 +143,7 @@ namespace edm { productSelectorRules, inputType, branchIDListHelper, + nullptr, thinnedAssociationsHelper, associationsFromSecondary, nullptr, @@ -148,6 +158,7 @@ namespace edm { enablePrefetching, enforceGUIDInFileName) {} + // Constructor used by RootEmbeddedFileSequence RootFile(std::string const& fileName, ProcessConfiguration const& processConfiguration, std::string const& logicalFileName, @@ -185,6 +196,7 @@ namespace edm { nullptr, nullptr, nullptr, + nullptr, false, processHistoryRegistry, indexesIntoFiles, @@ -209,6 +221,13 @@ namespace edm { std::shared_ptr readLuminosityBlockAuxiliary_(); std::shared_ptr readRunAuxiliary_(); std::shared_ptr readFakeRunAuxiliary_(); + + void fillProcessBlockHelper_(); + bool initializeFirstProcessBlockEntry(); + bool endOfProcessBlocksReached() const; + bool nextProcessBlock_(ProcessBlockPrincipal&); + void readProcessBlock_(ProcessBlockPrincipal&); + void readRun_(RunPrincipal& runPrincipal); void readFakeRun_(RunPrincipal& runPrincipal); void readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal); @@ -225,7 +244,9 @@ namespace edm { std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; } bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; } - std::unique_ptr createFileBlock() const; + std::shared_ptr createFileBlock(); + void updateFileBlock(FileBlock&); + bool setEntryAtItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) { return (event != 0) ? setEntryAtEvent(run, lumi, event) : (lumi ? setEntryAtLumi(run, lumi) : setEntryAtRun(run)); } @@ -241,6 +262,10 @@ namespace edm { eventTree_.rewind(); lumiTree_.rewind(); runTree_.rewind(); + currentProcessBlockTree_ = 0; + for (auto& processBlockTree : processBlockTrees_) { + processBlockTree->rewindToInvalid(); + } } void setToLastEntry() { indexIntoFileIter_ = indexIntoFileEnd_; } @@ -268,13 +293,18 @@ namespace edm { signalslot::Signal const* postEventReadSource); private: - RootTreePtrArray& treePointers() { return treePointers_; } + void makeProcessBlockRootTrees(std::shared_ptr filePtr, + int treeMaxVirtualSize, + bool enablePrefetching, + InputType inputType, + StoredProcessBlockHelper const& storedProcessBlockHelper); bool skipThisEntry(); void setIfFastClonable(int remainingEvents, int remainingLumis); void validateFile(InputType inputType, bool usingGoToEvent); void fillIndexIntoFile(); EventAuxiliary fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry); EventAuxiliary const& fillThisEventAuxiliary(); + void fillEventToProcessBlockIndexes(); bool fillEventHistory(EventAuxiliary& evtAux, EventSelectionIDVector& eventSelectionIDs, BranchListIndexes& branchListIndexes, @@ -282,11 +312,21 @@ namespace edm { std::shared_ptr fillLumiAuxiliary(); std::shared_ptr fillRunAuxiliary(); std::string const& newBranchToOldBranch(std::string const& newBranch) const; + void setPresenceInProductRegistry(ProductRegistry&, StoredProcessBlockHelper const&); void markBranchToBeDropped(bool dropDescendants, BranchDescription const& branch, std::set& branchesToDrop, std::map const& droppedToKeptAlias) const; - void dropOnInput(ProductRegistry& reg, ProductSelectorRules const& rules, bool dropDescendants, InputType inputType); + void dropOnInputAndReorder(ProductRegistry&, + ProductSelectorRules const&, + bool dropDescendants, + InputType, + StoredProcessBlockHelper&, + ProcessBlockHelper const*); + void dropProcessesAndReorder(StoredProcessBlockHelper&, + std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelper const*); + void readParentageTree(InputType inputType); void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType); // backward compatibility @@ -340,18 +380,24 @@ namespace edm { RootTree eventTree_; RootTree lumiTree_; RootTree runTree_; - RootTreePtrArray treePointers_; + std::vector>> processBlockTrees_; + unsigned int currentProcessBlockTree_ = 0; + std::vector> treePointers_; //Should only be used by fillThisEventAuxiliary() IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_; std::shared_ptr productRegistry_; std::shared_ptr branchIDLists_; edm::propagate_const> branchIDListHelper_; + edm::propagate_const processBlockHelper_; + edm::propagate_const> storedProcessBlockHelper_; edm::propagate_const> fileThinnedAssociationsHelper_; edm::propagate_const> thinnedAssociationsHelper_; InputSource::ProcessingMode processingMode_; edm::propagate_const runHelper_; std::map newBranchToOldBranch_; - edm::propagate_const eventHistoryTree_; // backward compatibility + edm::propagate_const eventHistoryTree_; // backward compatibility + EventToProcessBlockIndexes eventToProcessBlockIndexes_; + edm::propagate_const eventToProcessBlockIndexesBranch_; edm::propagate_const> history_; // backward compatibility edm::propagate_const> branchChildren_; edm::propagate_const> duplicateChecker_; diff --git a/IOPool/Input/src/RootInputFileSequence.cc b/IOPool/Input/src/RootInputFileSequence.cc index eb4f3f5c60432..616a67944b0ac 100644 --- a/IOPool/Input/src/RootInputFileSequence.cc +++ b/IOPool/Input/src/RootInputFileSequence.cc @@ -63,6 +63,21 @@ namespace edm { rootFile()->readRun_(runPrincipal); } + void RootInputFileSequence::fillProcessBlockHelper_() { + assert(rootFile()); + return rootFile()->fillProcessBlockHelper_(); + } + + bool RootInputFileSequence::nextProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(rootFile()); + return rootFile()->nextProcessBlock_(processBlockPrincipal); + } + + void RootInputFileSequence::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(rootFile()); + rootFile()->readProcessBlock_(processBlockPrincipal); + } + void RootInputFileSequence::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) { assert(rootFile()); rootFile()->readLuminosityBlock_(lumiPrincipal); diff --git a/IOPool/Input/src/RootInputFileSequence.h b/IOPool/Input/src/RootInputFileSequence.h index a7e902e3bc159..3fc18af5d0a21 100644 --- a/IOPool/Input/src/RootInputFileSequence.h +++ b/IOPool/Input/src/RootInputFileSequence.h @@ -43,6 +43,9 @@ namespace edm { void readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal); std::shared_ptr readRunAuxiliary_(); void readRun_(RunPrincipal& runPrincipal); + void fillProcessBlockHelper_(); + bool nextProcessBlock_(ProcessBlockPrincipal&); + void readProcessBlock_(ProcessBlockPrincipal&); bool skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, diff --git a/IOPool/Input/src/RootPrimaryFileSequence.cc b/IOPool/Input/src/RootPrimaryFileSequence.cc index a08b15cb4b7cf..e502314f0569b 100644 --- a/IOPool/Input/src/RootPrimaryFileSequence.cc +++ b/IOPool/Input/src/RootPrimaryFileSequence.cc @@ -62,7 +62,7 @@ namespace edm { if (rootFile()) { input_.productRegistryUpdate().updateFromInput(rootFile()->productRegistry()->productList()); if (initialNumberOfEventsToSkip_ != 0) { - skipEvents(initialNumberOfEventsToSkip_); + skipEventsAtBeginning(initialNumberOfEventsToSkip_); } } } @@ -71,24 +71,47 @@ namespace edm { void RootPrimaryFileSequence::endJob() { closeFile_(); } - std::unique_ptr RootPrimaryFileSequence::readFile_() { + std::shared_ptr RootPrimaryFileSequence::readFile_() { + std::shared_ptr fileBlock; if (firstFile_) { - // The first input file has already been opened. firstFile_ = false; + // Usually the first input file will already be open if (!rootFile()) { initFile(input_.skipBadFiles()); } + } else if (goToEventInNewFile_) { + goToEventInNewFile_ = false; + setAtFileSequenceNumber(goToFileSequenceOffset_); + initFile(false); + assert(rootFile()); + bool found = rootFile()->goToEvent(goToEventID_); + assert(found); + } else if (skipIntoNewFile_) { + skipIntoNewFile_ = false; + setAtFileSequenceNumber(skipToFileSequenceNumber_); + initFile(false); + assert(rootFile()); + if (skipToOffsetInFinalFile_ < 0) { + rootFile()->setToLastEntry(); + } + bool atEnd = rootFile()->skipEvents(skipToOffsetInFinalFile_); + assert(!atEnd && skipToOffsetInFinalFile_ == 0); } else { if (!nextFile()) { // handle case with last file bad and // skipBadFiles true - return std::unique_ptr(); + fb_ = fileBlock; + return fileBlock; } } if (!rootFile()) { - return std::make_unique(); + fileBlock = std::make_shared(); + fb_ = fileBlock; + return fileBlock; } - return rootFile()->createFileBlock(); + fileBlock = rootFile()->createFileBlock(); + fb_ = fileBlock; + return fileBlock; } void RootPrimaryFileSequence::closeFile_() { @@ -130,6 +153,7 @@ namespace edm { input_.productSelectorRules(), InputType::Primary, input_.branchIDListHelper(), + input_.processBlockHelper().get(), input_.thinnedAssociationsHelper(), nullptr, // associationsFromSecondary duplicateChecker(), @@ -162,10 +186,6 @@ namespace edm { assert(input_.skipBadFiles()); } while (true); - if (not rootFile()) { - return false; - } - // make sure the new product registry is compatible with the main one std::string mergeInfo = input_.productRegistryUpdate().merge(*rootFile()->productRegistry(), fileNames()[0], branchesMustMatch_); @@ -199,10 +219,11 @@ namespace edm { InputSource::ItemType RootPrimaryFileSequence::getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event) { - if (noMoreFiles()) { + if (noMoreFiles() || skipToStop_) { + skipToStop_ = false; return InputSource::IsStop; } - if (firstFile_) { + if (firstFile_ || goToEventInNewFile_ || skipIntoNewFile_) { return InputSource::IsFile; } if (rootFile()) { @@ -233,9 +254,12 @@ namespace edm { } rewindFile(); firstFile_ = true; + goToEventInNewFile_ = false; + skipIntoNewFile_ = false; + skipToStop_ = false; if (rootFile()) { if (initialNumberOfEventsToSkip_ != 0) { - skipEvents(initialNumberOfEventsToSkip_); + skipEventsAtBeginning(initialNumberOfEventsToSkip_); } } } @@ -246,20 +270,77 @@ namespace edm { rootFile()->rewind(); } - // Advance "offset" events. Offset can be positive or negative (or zero). - bool RootPrimaryFileSequence::skipEvents(int offset) { + // Advance "offset" events. Offset will be positive. + void RootPrimaryFileSequence::skipEventsAtBeginning(int offset) { assert(rootFile()); + assert(offset >= 0); while (offset != 0) { bool atEnd = rootFile()->skipEvents(offset); if ((offset > 0 || atEnd) && !nextFile()) { - return false; + return; } - if (offset < 0 && !previousFile()) { - setNoMoreFiles(); - return false; + } + } + + // Advance "offset" events. Offset can be positive or negative (or zero). + void RootPrimaryFileSequence::skipEvents(int offset) { + assert(rootFile()); + + bool atEnd = rootFile()->skipEvents(offset); + if (!atEnd && offset == 0) { + // successfully completed skip within current file + return; + } + + // Return, if without closing the current file we know the skip cannot be completed + skipToStop_ = false; + if (offset > 0 || atEnd) { + if (atLastFile() || noMoreFiles()) { + skipToStop_ = true; + return; } } - return true; + if (offset < 0 && atFirstFile()) { + skipToStop_ = true; + return; + } + + // Save the current file and position so that we can restore them + size_t const originalFileSequenceNumber = sequenceNumberOfFile(); + IndexIntoFile::IndexIntoFileItr originalPosition = rootFile()->indexIntoFileIter(); + + if ((offset > 0 || atEnd) && !nextFile()) { + skipToStop_ = true; // Can only get here if skipBadFiles is true + } + if (offset < 0 && !previousFile()) { + skipToStop_ = true; // Can't actually get here + } + + if (!skipToStop_) { + while (offset != 0) { + skipToOffsetInFinalFile_ = offset; + bool atEnd = rootFile()->skipEvents(offset); + if ((offset > 0 || atEnd) && !nextFile()) { + skipToStop_ = true; + break; + } + if (offset < 0 && !previousFile()) { + skipToStop_ = true; + break; + } + } + if (!skipToStop_) { + skipIntoNewFile_ = true; + } + } + skipToFileSequenceNumber_ = sequenceNumberOfFile(); + + // Restore the original file and position + setAtFileSequenceNumber(originalFileSequenceNumber); + initFile(false); + assert(rootFile()); + rootFile()->setPosition(originalPosition); + rootFile()->updateFileBlock(*fb_); } bool RootPrimaryFileSequence::goToEvent(EventID const& eventID) { @@ -272,36 +353,34 @@ namespace edm { if (rootFile() && indexesIntoFiles().size() == 1) { return false; } - // Save the current file and position so that we can restore them - // if we fail to restore the desired event - bool closedOriginalFile = false; - size_t const originalFileSequenceNumber = sequenceNumberOfFile(); - IndexIntoFile::IndexIntoFileItr originalPosition = rootFile()->indexIntoFileIter(); - // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files. for (auto it = indexesIntoFiles().begin(), itEnd = indexesIntoFiles().end(); it != itEnd; ++it) { if (*it && (*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) { - // We found it. Close the currently open file, and open the correct one. - setAtFileSequenceNumber(it - indexesIntoFiles().begin()); - initFile(false); - // Now get the item from the correct file. - assert(rootFile()); - bool found = rootFile()->goToEvent(eventID); - assert(found); + goToEventInNewFile_ = true; + goToFileSequenceOffset_ = it - indexesIntoFiles().begin(); + goToEventID_ = eventID; return true; } } + + // Save the current file and position so that we can restore them + bool closedOriginalFile = false; + size_t const originalFileSequenceNumber = sequenceNumberOfFile(); + IndexIntoFile::IndexIntoFileItr originalPosition = rootFile()->indexIntoFileIter(); + // Look for item in files not yet opened. + bool foundIt = false; for (auto it = indexesIntoFiles().begin(), itEnd = indexesIntoFiles().end(); it != itEnd; ++it) { if (!*it) { setAtFileSequenceNumber(it - indexesIntoFiles().begin()); initFile(false); + assert(rootFile()); closedOriginalFile = true; if ((*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) { - assert(rootFile()); - if (rootFile()->goToEvent(eventID)) { - return true; - } + foundIt = true; + goToEventInNewFile_ = true; + goToFileSequenceOffset_ = it - indexesIntoFiles().begin(); + goToEventID_ = eventID; } } } @@ -310,7 +389,9 @@ namespace edm { initFile(false); assert(rootFile()); rootFile()->setPosition(originalPosition); + rootFile()->updateFileBlock(*fb_); } + return foundIt; } return false; } diff --git a/IOPool/Input/src/RootPrimaryFileSequence.h b/IOPool/Input/src/RootPrimaryFileSequence.h index 903e1001c82f5..339c22561b962 100644 --- a/IOPool/Input/src/RootPrimaryFileSequence.h +++ b/IOPool/Input/src/RootPrimaryFileSequence.h @@ -39,11 +39,12 @@ namespace edm { RootPrimaryFileSequence(RootPrimaryFileSequence const&) = delete; // Disallow copying and moving RootPrimaryFileSequence& operator=(RootPrimaryFileSequence const&) = delete; // Disallow copying and moving - std::unique_ptr readFile_(); + std::shared_ptr readFile_(); void closeFile_() override; void endJob(); InputSource::ItemType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event); - bool skipEvents(int offset); + void skipEventsAtBeginning(int offset); + void skipEvents(int offset); bool goToEvent(EventID const& eventID); void rewind_(); static void fillDescription(ParameterSetDescription& desc); @@ -70,12 +71,20 @@ namespace edm { std::shared_ptr duplicateChecker() const { return get_underlying_safe(duplicateChecker_); } std::shared_ptr& duplicateChecker() { return get_underlying_safe(duplicateChecker_); } + edm::propagate_const> fb_; edm::propagate_const> eventSkipperByID_; int initialNumberOfEventsToSkip_; bool noEventSort_; unsigned int treeCacheSize_; edm::propagate_const> duplicateChecker_; bool usingGoToEvent_; + bool goToEventInNewFile_ = false; + size_t goToFileSequenceOffset_ = 0; + EventID goToEventID_; + bool skipToStop_ = false; + bool skipIntoNewFile_ = false; + size_t skipToFileSequenceNumber_ = 0; + int skipToOffsetInFinalFile_ = 0; bool enablePrefetching_; bool enforceGUIDInFileName_; }; // class RootPrimaryFileSequence diff --git a/IOPool/Input/src/RootTree.cc b/IOPool/Input/src/RootTree.cc index e503aac175e2b..d3f6f251e70d5 100644 --- a/IOPool/Input/src/RootTree.cc +++ b/IOPool/Input/src/RootTree.cc @@ -2,15 +2,13 @@ #include "RootDelayedReader.h" #include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/Exception.h" -#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/Provenance/interface/BranchType.h" #include "InputFile.h" #include "TTree.h" -#include "TTreeIndex.h" #include "TTreeCache.h" #include "TLeaf.h" #include -#include namespace edm { namespace { @@ -26,60 +24,75 @@ namespace edm { return branch; } } // namespace + + // Used for all RootTrees + // All the other constructors delegate to this one RootTree::RootTree(std::shared_ptr filePtr, BranchType const& branchType, unsigned int nIndexes, - unsigned int maxVirtualSize, - unsigned int cacheSize, unsigned int learningEntries, bool enablePrefetching, InputType inputType) : filePtr_(filePtr), - tree_(dynamic_cast( - filePtr_.get() != nullptr ? filePtr_->Get(BranchTypeToProductTreeName(branchType).c_str()) : nullptr)), - metaTree_(dynamic_cast( - filePtr_.get() != nullptr ? filePtr_->Get(BranchTypeToMetaDataTreeName(branchType).c_str()) : nullptr)), branchType_(branchType), - auxBranch_(tree_ ? getAuxiliaryBranch(tree_, branchType_) : nullptr), - treeCache_(), - rawTreeCache_(), - triggerTreeCache_(), - rawTriggerTreeCache_(), - trainedSet_(), - triggerSet_(), - entries_(tree_ ? tree_->GetEntries() : 0), - entryNumber_(-1), - entryNumberForIndex_(new std::vector(nIndexes, IndexIntoFile::invalidEntry)), - branchNames_(), - branches_{}, - trainNow_(false), - switchOverEntry_(-1), - rawTriggerSwitchOverEntry_(-1), - performedSwitchOver_{false}, + entryNumberForIndex_(std::make_unique>(nIndexes, IndexIntoFile::invalidEntry)), learningEntries_(learningEntries), - cacheSize_(cacheSize), - treeAutoFlush_(0), enablePrefetching_(enablePrefetching), enableTriggerCache_(branchType_ == InEvent), - rootDelayedReader_(new RootDelayedReader(*this, filePtr, inputType)), - branchEntryInfoBranch_(metaTree_ ? getProductProvenanceBranch(metaTree_, branchType_) - : (tree_ ? getProductProvenanceBranch(tree_, branchType_) : nullptr)), - infoTree_(dynamic_cast(filePtr_.get() != nullptr - ? filePtr->Get(BranchTypeToInfoTreeName(branchType).c_str()) - : nullptr)) // backward compatibility - { + rootDelayedReader_(std::make_unique(*this, filePtr, inputType)) {} + + // Used for Event/Lumi/Run RootTrees + RootTree::RootTree(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + unsigned int maxVirtualSize, + unsigned int cacheSize, + unsigned int learningEntries, + bool enablePrefetching, + InputType inputType) + : RootTree(filePtr, branchType, nIndexes, learningEntries, enablePrefetching, inputType) { + init(BranchTypeToProductTreeName(branchType), maxVirtualSize, cacheSize); + metaTree_ = dynamic_cast(filePtr_->Get(BranchTypeToMetaDataTreeName(branchType).c_str())); + auxBranch_ = getAuxiliaryBranch(tree_, branchType_); + branchEntryInfoBranch_ = + metaTree_ ? getProductProvenanceBranch(metaTree_, branchType_) : getProductProvenanceBranch(tree_, branchType_); + infoTree_ = + dynamic_cast(filePtr->Get(BranchTypeToInfoTreeName(branchType).c_str())); // backward compatibility + } + + // Used for ProcessBlock RootTrees + RootTree::RootTree(std::shared_ptr filePtr, + BranchType const& branchType, + std::string const& processName, + unsigned int nIndexes, + unsigned int maxVirtualSize, + unsigned int cacheSize, + unsigned int learningEntries, + bool enablePrefetching, + InputType inputType) + : RootTree(filePtr, branchType, nIndexes, learningEntries, enablePrefetching, inputType) { + processName_ = processName; + init(BranchTypeToProductTreeName(branchType, processName), maxVirtualSize, cacheSize); + } + + void RootTree::init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize) { + if (filePtr_.get() != nullptr) { + tree_ = dynamic_cast(filePtr_->Get(productTreeName.c_str())); + } if (not tree_) { throw cms::Exception("WrongFileFormat") - << "The ROOT file does not contain a TTree named " << BranchTypeToProductTreeName(branchType) + << "The ROOT file does not contain a TTree named " << productTreeName << "\n This is either not an edm ROOT file or is one that has been corrupted."; } + entries_ = tree_->GetEntries(); + // On merged files in older releases of ROOT, the autoFlush setting is always negative; we must guess. // TODO: On newer merged files, we should be able to get this from the cluster iterator. - long treeAutoFlush = (tree_ ? tree_->GetAutoFlush() : 0); + long treeAutoFlush = tree_->GetAutoFlush(); if (treeAutoFlush < 0) { // The "+1" is here to avoid divide-by-zero in degenerate cases. Long64_t averageEventSizeBytes = tree_->GetZipBytes() / (tree_->GetEntries() + 1) + 1; - treeAutoFlush_ = cacheSize_ / averageEventSizeBytes + 1; + treeAutoFlush_ = cacheSize / averageEventSizeBytes + 1; } else { treeAutoFlush_ = treeAutoFlush; } @@ -88,7 +101,7 @@ namespace edm { } setTreeMaxVirtualSize(maxVirtualSize); setCacheSize(cacheSize); - if (tree_) { + if (branchType_ == InEvent) { Int_t branchCount = tree_->GetListOfBranches()->GetEntriesFast(); trainedSet_.reserve(branchCount); triggerSet_.reserve(branchCount); @@ -108,9 +121,15 @@ namespace edm { } bool RootTree::isValid() const { + // ProcessBlock + if (branchType_ == InProcess) { + return tree_ != nullptr; + } + // Run/Lumi/Event if (metaTree_ == nullptr || metaTree_->GetNbranches() == 0) { return tree_ != nullptr && auxBranch_ != nullptr; } + // Backward compatibility for Run/Lumi/Event if (tree_ != nullptr && auxBranch_ != nullptr && metaTree_ != nullptr) { // backward compatibility if (branchEntryInfoBranch_ != nullptr || infoTree_ != nullptr) return true; // backward compatibility @@ -473,13 +492,15 @@ namespace edm { // so that ROOT does not also delete it. filePtr_->SetCacheRead(nullptr); - // Must also manually add things to the trained set. - TObjArray* branches = tree_->GetListOfBranches(); - int branchCount = branches->GetEntriesFast(); - for (int i = 0; i < branchCount; i++) { - TBranch* branch = (TBranch*)branches->UncheckedAt(i); - if ((branchNames[0] == '*') || (strcmp(branchNames, branch->GetName()) == 0)) { - trainedSet_.insert(branch); + if (branchType_ == InEvent) { + // Must also manually add things to the trained set. + TObjArray* branches = tree_->GetListOfBranches(); + int branchCount = branches->GetEntriesFast(); + for (int i = 0; i < branchCount; i++) { + TBranch* branch = (TBranch*)branches->UncheckedAt(i); + if ((branchNames[0] == '*') || (strcmp(branchNames, branch->GetName()) == 0)) { + trainedSet_.insert(branch); + } } } } diff --git a/IOPool/Input/src/RootTree.h b/IOPool/Input/src/RootTree.h index a5050267fa94c..be851c8973efd 100644 --- a/IOPool/Input/src/RootTree.h +++ b/IOPool/Input/src/RootTree.h @@ -8,11 +8,13 @@ RootTree.h // used by ROOT input sources ----------------------------------------------------------------------*/ #include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/Provenance/interface/BranchID.h" #include "DataFormats/Provenance/interface/IndexIntoFile.h" -#include "DataFormats/Provenance/interface/BranchKey.h" -#include "DataFormats/Provenance/interface/ProvenanceFwd.h" #include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/Signal.h" #include "FWCore/Utilities/interface/thread_safety_macros.h" #include "Rtypes.h" @@ -24,31 +26,20 @@ RootTree.h // used by ROOT input sources #include #include -class TBranch; class TClass; class TTree; class TTreeCache; namespace edm { - class BranchKey; class RootDelayedReader; class InputFile; - class RootTree; - - class StreamContext; - class ModuleCallingContext; - - namespace signalslot { - template - class Signal; - } namespace roottree { unsigned int const defaultCacheSize = 20U * 1024 * 1024; unsigned int const defaultNonEventCacheSize = 1U * 1024 * 1024; unsigned int const defaultLearningEntries = 20U; unsigned int const defaultNonEventLearningEntries = 1U; - typedef IndexIntoFile::EntryNumber_t EntryNumber; + using EntryNumber = IndexIntoFile::EntryNumber_t; struct BranchInfo { BranchInfo(BranchDescription const& prod) : branchDescription_(prod), productBranch_(nullptr), classCache_(nullptr), offsetToWrapperBase_(0) {} @@ -61,7 +52,9 @@ namespace edm { class BranchMap { public: - void reserve(size_t iSize) { map_.reserve(iSize); } + using Map = std::unordered_map; + + void reserve(Map::size_type iSize) { map_.reserve(iSize); } void insert(edm::BranchID const& iKey, BranchInfo const& iInfo) { map_.emplace(iKey.id(), iInfo); } BranchInfo const* find(BranchID const& iKey) const { auto itFound = map_.find(iKey.id()); @@ -72,7 +65,7 @@ namespace edm { } private: - std::unordered_map map_; + Map map_; }; Int_t getEntry(TBranch* branch, EntryNumber entryNumber); @@ -85,8 +78,16 @@ namespace edm { class RootTree { public: - typedef roottree::BranchMap BranchMap; - typedef roottree::EntryNumber EntryNumber; + using BranchMap = roottree::BranchMap; + using EntryNumber = roottree::EntryNumber; + + RootTree(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + unsigned int learningEntries, + bool enablePrefetching, + InputType inputType); + RootTree(std::shared_ptr filePtr, BranchType const& branchType, unsigned int nIndexes, @@ -95,13 +96,26 @@ namespace edm { unsigned int learningEntries, bool enablePrefetching, InputType inputType); + + RootTree(std::shared_ptr filePtr, + BranchType const& branchType, + std::string const& processName, + unsigned int nIndexes, + unsigned int maxVirtualSize, + unsigned int cacheSize, + unsigned int learningEntries, + bool enablePrefetching, + InputType inputType); + + void init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize); + ~RootTree(); RootTree(RootTree const&) = delete; // Disallow copying and moving RootTree& operator=(RootTree const&) = delete; // Disallow copying and moving bool isValid() const; - void numberOfBranchesToAdd(size_t iSize) { branches_.reserve(iSize); } + void numberOfBranchesToAdd(BranchMap::Map::size_type iSize) { branches_.reserve(iSize); } void addBranch(BranchDescription const& prod, std::string const& oldBranchName); void dropBranch(std::string const& oldBranchName); void getEntry(TBranch* branch, EntryNumber entry) const; @@ -112,6 +126,7 @@ namespace edm { bool current() const { return entryNumber_ < entries_ && entryNumber_ >= 0; } bool current(EntryNumber entry) const { return entry < entries_ && entry >= 0; } void rewind() { entryNumber_ = 0; } + void rewindToInvalid() { entryNumber_ = IndexIntoFile::invalidEntry; } void close(); bool skipEntries(unsigned int& offset); EntryNumber const& entryNumber() const { return entryNumber_; } @@ -154,6 +169,7 @@ namespace edm { TTree const* tree() const { return tree_; } TTree* tree() { return tree_; } TTree const* metaTree() const { return metaTree_; } + TTree* metaTree() { return metaTree_; } BranchMap const& branches() const; //For backwards compatibility @@ -166,6 +182,7 @@ namespace edm { void resetTraining() { trainNow_ = true; } BranchType branchType() const { return branchType_; } + std::string const& processName() const { return processName_; } void setSignals( signalslot::Signal const* preEventReadSource, @@ -181,10 +198,11 @@ namespace edm { // We use bare pointers for pointers to some ROOT entities. // Root owns them and uses bare pointers internally. // Therefore,using smart pointers here will do no good. - TTree* tree_; - TTree* metaTree_; + TTree* tree_ = nullptr; + TTree* metaTree_ = nullptr; BranchType branchType_; - TBranch* auxBranch_; + std::string processName_; + TBranch* auxBranch_ = nullptr; // We use a smart pointer to own the TTreeCache. // Unfortunately, ROOT owns it when attached to a TFile, but not after it is detached. // So, we make sure to it is detached before closing the TFile so there is no double delete. @@ -195,27 +213,27 @@ namespace edm { CMS_SA_ALLOW mutable std::shared_ptr rawTriggerTreeCache_; CMS_SA_ALLOW mutable std::unordered_set trainedSet_; CMS_SA_ALLOW mutable std::unordered_set triggerSet_; - EntryNumber entries_; - EntryNumber entryNumber_; + EntryNumber entries_ = 0; + EntryNumber entryNumber_ = IndexIntoFile::invalidEntry; std::unique_ptr > entryNumberForIndex_; std::vector branchNames_; BranchMap branches_; - bool trainNow_; - EntryNumber switchOverEntry_; - CMS_SA_ALLOW mutable EntryNumber rawTriggerSwitchOverEntry_; - CMS_SA_ALLOW mutable bool performedSwitchOver_; + bool trainNow_ = false; + EntryNumber switchOverEntry_ = -1; + CMS_SA_ALLOW mutable EntryNumber rawTriggerSwitchOverEntry_ = -1; + CMS_SA_ALLOW mutable bool performedSwitchOver_ = false; unsigned int learningEntries_; - unsigned int cacheSize_; - unsigned long treeAutoFlush_; + unsigned int cacheSize_ = 0; + unsigned long treeAutoFlush_ = 0; // Enable asynchronous I/O in ROOT (done in a separate thread). Only takes // effect on the primary treeCache_; all other caches have this explicitly disabled. bool enablePrefetching_; bool enableTriggerCache_; std::unique_ptr rootDelayedReader_; - TBranch* branchEntryInfoBranch_; //backwards compatibility + TBranch* branchEntryInfoBranch_ = nullptr; //backwards compatibility // below for backward compatibility - TTree* infoTree_; // backward compatibility + TTree* infoTree_ = nullptr; // backward compatibility }; } // namespace edm #endif diff --git a/IOPool/Input/test/PoolInputTest_skip_with_failure_cfg.py b/IOPool/Input/test/PoolInputTest_skip_with_failure_cfg.py index 2ebfda51a3472..8bdf9bc17ef76 100644 --- a/IOPool/Input/test/PoolInputTest_skip_with_failure_cfg.py +++ b/IOPool/Input/test/PoolInputTest_skip_with_failure_cfg.py @@ -1,5 +1,3 @@ -# The following comments couldn't be translated into the new config version: - # Configuration file for PoolInputTest import FWCore.ParameterSet.Config as cms diff --git a/IOPool/Output/interface/PoolOutputModule.h b/IOPool/Output/interface/PoolOutputModule.h index 80797eea85a88..023bd47235837 100644 --- a/IOPool/Output/interface/PoolOutputModule.h +++ b/IOPool/Output/interface/PoolOutputModule.h @@ -21,9 +21,11 @@ #include "IOPool/Common/interface/RootServiceChecker.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/one/OutputModule.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/propagate_const.h" #include "DataFormats/Provenance/interface/BranchChildren.h" #include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/BranchType.h" #include "DataFormats/Provenance/interface/ParentageID.h" class TTree; @@ -74,7 +76,7 @@ namespace edm { ~AuxItem() {} int basketSize_; }; - typedef std::array AuxItemArray; + using AuxItemArray = std::array; AuxItemArray const& auxItems() const { return auxItems_; } struct OutputItem { @@ -110,9 +112,7 @@ namespace edm { int basketSize_; }; - typedef std::vector OutputItemList; - - typedef std::array OutputItemListArray; + using OutputItemList = std::vector; struct SpecialSplitLevelForBranch { SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel) @@ -126,9 +126,9 @@ namespace edm { int splitLevel_; }; - OutputItemListArray const& selectedOutputItemList() const { return selectedOutputItemList_; } + std::vector const& selectedOutputItemList() const { return selectedOutputItemList_; } - OutputItemListArray& selectedOutputItemList() { return selectedOutputItemList_; } + std::vector& selectedOutputItemList() { return selectedOutputItemList_; } BranchChildren const& branchChildren() const { return branchChildren_; } @@ -148,8 +148,9 @@ namespace edm { void openFile(FileBlock const& fb) override; void respondToOpenInputFile(FileBlock const& fb) override; void respondToCloseInputFile(FileBlock const& fb) override; - void writeLuminosityBlock(LuminosityBlockForOutput const& lb) override; - void writeRun(RunForOutput const& r) override; + void writeLuminosityBlock(LuminosityBlockForOutput const&) override; + void writeRun(RunForOutput const&) override; + void writeProcessBlock(ProcessBlockForOutput const&) override; bool isFileOpen() const override; void reallyOpenFile(); void reallyCloseFile() override; @@ -157,7 +158,7 @@ namespace edm { void setProcessesWithSelectedMergeableRunProducts(std::set const&) override; - typedef std::map> BranchParents; + using BranchParents = std::map>; void updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, BranchID const& branchID); void updateBranchParents(EventForOutput const& e); void fillDependencyGraph(); @@ -175,14 +176,18 @@ namespace edm { void writeThinnedAssociationsHelper(); void writeProductDependencies(); void writeEventAuxiliary(); + void writeProcessBlockHelper(); void finishEndFile(); - void fillSelectedItemList(BranchType branchtype, TTree* theInputTree); + void fillSelectedItemList(BranchType branchtype, + std::string const& processName, + TTree* theInputTree, + OutputItemList&); void beginInputFile(FileBlock const& fb); RootServiceChecker rootServiceChecker_; AuxItemArray auxItems_; - OutputItemListArray selectedOutputItemList_; + std::vector selectedOutputItemList_; std::vector specialSplitLevelForBranches_; std::string const fileName_; std::string const logicalFileName_; diff --git a/IOPool/Output/src/PoolOutputModule.cc b/IOPool/Output/src/PoolOutputModule.cc index 4574e1c3dd478..cfd8c3275bc6d 100644 --- a/IOPool/Output/src/PoolOutputModule.cc +++ b/IOPool/Output/src/PoolOutputModule.cc @@ -168,23 +168,28 @@ namespace edm { return std::regex(tmp); } - void PoolOutputModule::fillSelectedItemList(BranchType branchType, TTree* theInputTree) { + void PoolOutputModule::fillSelectedItemList(BranchType branchType, + std::string const& processName, + TTree* theInputTree, + OutputItemList& outputItemList) { SelectedProducts const& keptVector = keptProducts()[branchType]; - OutputItemList& outputItemList = selectedOutputItemList_[branchType]; - AuxItem& auxItem = auxItems_[branchType]; - auto basketSize = (InEvent == branchType) ? eventAuxBasketSize_ : basketSize_; + if (branchType != InProcess) { + AuxItem& auxItem = auxItems_[branchType]; - // Fill AuxItem - if (theInputTree != nullptr && !overrideInputFileSplitLevels_) { - TBranch* auxBranch = theInputTree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); - if (auxBranch) { - auxItem.basketSize_ = auxBranch->GetBasketSize(); + auto basketSize = (InEvent == branchType) ? eventAuxBasketSize_ : basketSize_; + + // Fill AuxItem + if (theInputTree != nullptr && !overrideInputFileSplitLevels_) { + TBranch* auxBranch = theInputTree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); + if (auxBranch) { + auxItem.basketSize_ = auxBranch->GetBasketSize(); + } else { + auxItem.basketSize_ = basketSize; + } } else { auxItem.basketSize_ = basketSize; } - } else { - auxItem.basketSize_ = basketSize; } // Fill outputItemList with an entry for each branch. @@ -193,6 +198,9 @@ namespace edm { int basketSize = BranchDescription::invalidBasketSize; BranchDescription const& prod = *kept.first; + if (branchType == InProcess && processName != prod.processName()) { + continue; + } TBranch* theBranch = ((!prod.produced() && theInputTree != nullptr && !overrideInputFileSplitLevels_) ? theInputTree->GetBranch(prod.branchName().c_str()) : nullptr); @@ -240,15 +248,29 @@ namespace edm { void PoolOutputModule::respondToOpenInputFile(FileBlock const& fb) { if (!initializedFromInput_) { - for (int i = InEvent; i < NumBranchTypes; ++i) { - if (i == InProcess) { - // ProcessBlock output not implemented yet - continue; - } + std::vector const& processesWithProcessBlockProducts = + outputProcessBlockHelper().processesWithProcessBlockProducts(); + unsigned int numberOfProcessesWithProcessBlockProducts = processesWithProcessBlockProducts.size(); + unsigned int numberOfTTrees = numberOfRunLumiEventProductTrees + numberOfProcessesWithProcessBlockProducts; + selectedOutputItemList_.resize(numberOfTTrees); + + for (unsigned int i = InEvent; i < NumBranchTypes; ++i) { BranchType branchType = static_cast(i); - TTree* theInputTree = - (branchType == InEvent ? fb.tree() : (branchType == InLumi ? fb.lumiTree() : fb.runTree())); - fillSelectedItemList(branchType, theInputTree); + if (branchType != InProcess) { + std::string processName; + TTree* theInputTree = + (branchType == InEvent ? fb.tree() : (branchType == InLumi ? fb.lumiTree() : fb.runTree())); + OutputItemList& outputItemList = selectedOutputItemList_[branchType]; + fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + } else { + // Handle output items in ProcessBlocks + for (unsigned int k = InProcess; k < numberOfTTrees; ++k) { + OutputItemList& outputItemList = selectedOutputItemList_[k]; + std::string const& processName = processesWithProcessBlockProducts[k - InProcess]; + TTree* theInputTree = fb.processBlockTree(processName); + fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + } + } } initializedFromInput_ = true; } @@ -283,6 +305,8 @@ namespace edm { void PoolOutputModule::writeRun(RunForOutput const& r) { rootOutputFile_->writeRun(r); } + void PoolOutputModule::writeProcessBlock(ProcessBlockForOutput const& pb) { rootOutputFile_->writeProcessBlock(pb); } + void PoolOutputModule::reallyCloseFile() { writeEventAuxiliary(); fillDependencyGraph(); @@ -299,6 +323,7 @@ namespace edm { writeBranchIDListRegistry(); writeThinnedAssociationsHelper(); writeProductDependencies(); //branchChildren used here + writeProcessBlockHelper(); branchChildren_.clear(); finishEndFile(); @@ -322,6 +347,7 @@ namespace edm { void PoolOutputModule::writeThinnedAssociationsHelper() { rootOutputFile_->writeThinnedAssociationsHelper(); } void PoolOutputModule::writeProductDependencies() { rootOutputFile_->writeProductDependencies(); } void PoolOutputModule::writeEventAuxiliary() { rootOutputFile_->writeEventAuxiliary(); } + void PoolOutputModule::writeProcessBlockHelper() { rootOutputFile_->writeProcessBlockHelper(); } void PoolOutputModule::finishEndFile() { rootOutputFile_->finishEndFile(); rootOutputFile_ = nullptr; diff --git a/IOPool/Output/src/RootOutputFile.cc b/IOPool/Output/src/RootOutputFile.cc index cf51e7bc205cc..ecfb6c0360bef 100644 --- a/IOPool/Output/src/RootOutputFile.cc +++ b/IOPool/Output/src/RootOutputFile.cc @@ -4,16 +4,19 @@ #include "FWCore/Utilities/interface/GlobalIdentifier.h" #include "DataFormats/Provenance/interface/EventAuxiliary.h" +#include "DataFormats/Provenance/interface/BranchDescription.h" #include "FWCore/Version/interface/GetFileFormatVersion.h" #include "DataFormats/Provenance/interface/FileFormatVersion.h" #include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/Algorithms.h" #include "FWCore/Utilities/interface/Digest.h" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" #include "FWCore/Framework/interface/FileBlock.h" #include "FWCore/Framework/interface/EventForOutput.h" #include "FWCore/Framework/interface/LuminosityBlockForOutput.h" #include "FWCore/Framework/interface/MergeableRunProductMetadata.h" #include "FWCore/Framework/interface/OccurrenceForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" #include "FWCore/Framework/interface/RunForOutput.h" #include "FWCore/MessageLogger/interface/JobReport.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" @@ -23,10 +26,12 @@ #include "DataFormats/Provenance/interface/Parentage.h" #include "DataFormats/Provenance/interface/ParentageRegistry.h" #include "DataFormats/Provenance/interface/EventID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/ParameterSetBlob.h" #include "DataFormats/Provenance/interface/ParameterSetID.h" #include "DataFormats/Provenance/interface/ProcessHistoryID.h" #include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" #include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" #include "FWCore/Framework/interface/ConstProductRegistry.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" @@ -106,12 +111,18 @@ namespace edm { eventTree_(filePtr(), InEvent, om_->splitLevel(), om_->treeMaxVirtualSize()), lumiTree_(filePtr(), InLumi, om_->splitLevel(), om_->treeMaxVirtualSize()), runTree_(filePtr(), InRun, om_->splitLevel(), om_->treeMaxVirtualSize()), - treePointers_(), dataTypeReported_(false), processHistoryRegistry_(), parentageIDs_(), branchesWithStoredHistory_(), wrapperBaseTClass_(TClass::GetClass("edm::WrapperBase")) { + std::vector const& processesWithProcessBlockProducts = + om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); + for (auto const& processName : processesWithProcessBlockProducts) { + processBlockTrees_.emplace_back(std::make_unique( + filePtr(), InProcess, om_->splitLevel(), om_->treeMaxVirtualSize(), processName)); + } + if (om_->compressionAlgorithm() == std::string("ZLIB")) { filePtr_->SetCompressionAlgorithm(ROOT::kZLIB); } else if (om_->compressionAlgorithm() == std::string("LZMA")) { @@ -121,7 +132,7 @@ namespace edm { } else { throw Exception(errors::Configuration) << "PoolOutputModule configured with unknown compression algorithm '" << om_->compressionAlgorithm() << "'\n" - << "Allowed compression algorithms are ZLIB and LZMA\n"; + << "Allowed compression algorithms are ZLIB, LZMA, and ZSTD\n"; } if (-1 != om->eventAutoFlushSize()) { eventTree_.setAutoFlush(-1 * om->eventAutoFlushSize()); @@ -144,25 +155,28 @@ namespace edm { eventTree_.addAuxiliary( poolNames::branchListIndexesBranchName(), pBranchListIndexes_, om_->auxItems()[InEvent].basketSize_); + if (om_->outputProcessBlockHelper().productsFromInputKept()) { + eventTree_.addAuxiliary(poolNames::eventToProcessBlockIndexesBranchName(), + pEventToProcessBlockIndexes_, + om_->auxItems()[InEvent].basketSize_); + } + lumiTree_.addAuxiliary( BranchTypeToAuxiliaryBranchName(InLumi), pLumiAux_, om_->auxItems()[InLumi].basketSize_); runTree_.addAuxiliary( BranchTypeToAuxiliaryBranchName(InRun), pRunAux_, om_->auxItems()[InRun].basketSize_); - treePointers_[InEvent] = &eventTree_; - treePointers_[InLumi] = &lumiTree_; - treePointers_[InRun] = &runTree_; - treePointers_[InProcess] = nullptr; + treePointers_.emplace_back(&eventTree_); + treePointers_.emplace_back(&lumiTree_); + treePointers_.emplace_back(&runTree_); + for (auto& processBlockTree : processBlockTrees_) { + treePointers_.emplace_back(processBlockTree.get()); + } - for (int i = InEvent; i < NumBranchTypes; ++i) { - if (i == InProcess) { - // Output for ProcessBlocks is not implemented yet. - continue; - } - BranchType branchType = static_cast(i); - RootOutputTree* theTree = treePointers_[branchType]; - for (auto& item : om_->selectedOutputItemList()[branchType]) { + for (unsigned int i = 0; i < treePointers_.size(); ++i) { + RootOutputTree* theTree = treePointers_[i]; + for (auto& item : om_->selectedOutputItemList()[i]) { item.setProduct(nullptr); BranchDescription const& desc = *item.branchDescription(); theTree->addBranch(desc.branchName(), @@ -428,6 +442,7 @@ namespace edm { // must be set before calling fillBranches since they get written out in that routine. assert(pEventAux_->processHistoryID() == e.processHistoryID()); pBranchListIndexes_ = &e.branchListIndexes(); + pEventToProcessBlockIndexes_ = &e.eventToProcessBlockIndexes(); // Note: The EventSelectionIDVector should have a one to one correspondence with the processes in the process history. // Therefore, a new entry should be added if and only if the current process has been added to the process history, @@ -440,7 +455,8 @@ namespace edm { pEventSelectionIDs_ = &esids; ProductProvenanceRetriever const* provRetriever = e.productProvenanceRetrieverPtr(); assert(provRetriever); - fillBranches(InEvent, e, pEventEntryInfoVector_, provRetriever); + unsigned int ttreeIndex = InEvent; + fillBranches(InEvent, e, ttreeIndex, pEventEntryInfoVector_, provRetriever); // Add the dataType to the job report if it hasn't already been done if (!dataTypeReported_) { @@ -484,7 +500,8 @@ namespace edm { // Add lumi to index. indexIntoFile_.addEntry(reducedPHID, lumiAux_.run(), lumiAux_.luminosityBlock(), 0U, lumiEntryNumber_); ++lumiEntryNumber_; - fillBranches(InLumi, lb); + unsigned int ttreeIndex = InLumi; + fillBranches(InLumi, lb, ttreeIndex); lumiTree_.optimizeBaskets(10ULL * 1024 * 1024); Service reportSvc; @@ -506,13 +523,28 @@ namespace edm { indexIntoFile_.addEntry(reducedPHID, runAux_.run(), 0U, 0U, runEntryNumber_); r.mergeableRunProductMetadata()->addEntryToStoredMetadata(storedMergeableRunProductMetadata_); ++runEntryNumber_; - fillBranches(InRun, r); + unsigned int ttreeIndex = InRun; + fillBranches(InRun, r, ttreeIndex); runTree_.optimizeBaskets(10ULL * 1024 * 1024); Service reportSvc; reportSvc->reportRunNumber(reportToken_, r.run()); } + void RootOutputFile::writeProcessBlock(ProcessBlockForOutput const& pb) { + std::string const& processName = pb.processName(); + std::vector const& processesWithProcessBlockProducts = + om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); + std::vector::const_iterator it = + std::find(processesWithProcessBlockProducts.cbegin(), processesWithProcessBlockProducts.cend(), processName); + if (it == processesWithProcessBlockProducts.cend()) { + return; + } + unsigned int ttreeIndex = InProcess + std::distance(processesWithProcessBlockProducts.cbegin(), it); + fillBranches(InProcess, pb, ttreeIndex); + treePointers_[ttreeIndex]->optimizeBaskets(10ULL * 1024 * 1024); + } + void RootOutputFile::writeParentageRegistry() { Parentage const* desc(nullptr); @@ -602,7 +634,7 @@ namespace edm { void RootOutputFile::writeProductDescriptionRegistry() { // Make a local copy of the ProductRegistry, removing any transient or pruned products. - typedef ProductRegistry::ProductList ProductList; + using ProductList = ProductRegistry::ProductList; Service reg; ProductRegistry pReg(reg->productList()); ProductList& pList = const_cast(pReg.productList()); @@ -677,6 +709,20 @@ namespace edm { } } + void RootOutputFile::writeProcessBlockHelper() { + if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) { + StoredProcessBlockHelper storedProcessBlockHelper( + om_->outputProcessBlockHelper().processesWithProcessBlockProducts()); + om_->outputProcessBlockHelper().fillCacheIndices(storedProcessBlockHelper); + + StoredProcessBlockHelper* pStoredProcessBlockHelper = &storedProcessBlockHelper; + TBranch* b = metaDataTree_->Branch( + poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + } + void RootOutputFile::finishEndFile() { metaDataTree_->SetEntries(-1); RootOutputTree::writeTTree(metaDataTree_); @@ -685,26 +731,24 @@ namespace edm { RootOutputTree::writeTTree(parentageTree_); // Create branch aliases for all the branches in the - // events/lumis/runs trees. The loop is over all types of data - // products. - for (int i = InEvent; i < NumBranchTypes; ++i) { - if (i == InProcess) { - // Output for ProcessBlocks is not implemented yet. - continue; + // events/lumis/runs/processblock trees. The loop is over + // all types of data products. + for (unsigned int i = 0; i < treePointers_.size(); ++i) { + std::string processName; + BranchType branchType = InProcess; + if (i < InProcess) { + branchType = static_cast(i); + } else { + processName = om_->outputProcessBlockHelper().processesWithProcessBlockProducts()[i - InProcess]; } - BranchType branchType = static_cast(i); - setBranchAliases(treePointers_[branchType]->tree(), om_->keptProducts()[branchType]); - treePointers_[branchType]->writeTree(); + setBranchAliases(treePointers_[i]->tree(), om_->keptProducts()[branchType], processName); + treePointers_[i]->writeTree(); } // close the file -- mfp // Just to play it safe, zero all pointers to objects in the TFile to be closed. metaDataTree_ = parentageTree_ = nullptr; for (auto& treePointer : treePointers_) { - if (treePointer.get() == nullptr) { - // Output for ProcessBlock is not implemented yet - continue; - } treePointer->close(); treePointer = nullptr; } @@ -716,10 +760,15 @@ namespace edm { reportSvc->outputFileClosed(reportToken_); } - void RootOutputFile::setBranchAliases(TTree* tree, SelectedProducts const& branches) const { + void RootOutputFile::setBranchAliases(TTree* tree, + SelectedProducts const& branches, + std::string const& processName) const { if (tree && tree->GetNbranches() != 0) { for (auto const& selection : branches) { BranchDescription const& pd = *selection.first; + if (pd.branchType() == InProcess && processName != pd.processName()) { + continue; + } std::string const& full = pd.branchName() + "obj"; if (pd.branchAliases().empty()) { std::string const& alias = (pd.productInstanceName().empty() ? pd.moduleLabel() : pd.productInstanceName()); @@ -760,11 +809,12 @@ namespace edm { void RootOutputFile::fillBranches(BranchType const& branchType, OccurrenceForOutput const& occurrence, + unsigned int ttreeIndex, StoredProductProvenanceVector* productProvenanceVecPtr, ProductProvenanceRetriever const* provRetriever) { std::vector > dummies; - OutputItemList& items = om_->selectedOutputItemList()[branchType]; + OutputItemList& items = om_->selectedOutputItemList()[ttreeIndex]; bool const doProvenance = (productProvenanceVecPtr != nullptr) && (om_->dropMetaData() != PoolOutputModule::DropAll); @@ -794,7 +844,7 @@ namespace edm { bool produced = item.branchDescription()->produced(); bool getProd = - (produced || !fastCloning || treePointers_[branchType]->uncloned(item.branchDescription()->branchName())); + (produced || !fastCloning || treePointers_[ttreeIndex]->uncloned(item.branchDescription()->branchName())); bool keepProvenance = doProvenance && (produced || keepProvenanceForPrior); WrapperBase const* product = nullptr; @@ -829,7 +879,7 @@ namespace edm { if (doProvenance) productProvenanceVecPtr->assign(provenanceToKeep.begin(), provenanceToKeep.end()); - treePointers_[branchType]->fillTree(); + treePointers_[ttreeIndex]->fillTree(); if (doProvenance) productProvenanceVecPtr->clear(); } diff --git a/IOPool/Output/src/RootOutputFile.h b/IOPool/Output/src/RootOutputFile.h index 8a9a8565862c4..9eb35a1ab8f4d 100644 --- a/IOPool/Output/src/RootOutputFile.h +++ b/IOPool/Output/src/RootOutputFile.h @@ -18,7 +18,9 @@ #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/MessageLogger/interface/JobReport.h" +#include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" #include "DataFormats/Provenance/interface/BranchListIndex.h" #include "DataFormats/Provenance/interface/EventSelectionID.h" #include "DataFormats/Provenance/interface/FileID.h" @@ -45,9 +47,8 @@ namespace edm { class RootOutputFile { public: - typedef PoolOutputModule::OutputItem OutputItem; - typedef PoolOutputModule::OutputItemList OutputItemList; - typedef std::array, NumBranchTypes> RootOutputTreePtrArray; + using OutputItem = PoolOutputModule::OutputItem; + using OutputItemList = PoolOutputModule::OutputItemList; explicit RootOutputFile(PoolOutputModule* om, std::string const& fileName, std::string const& logicalFileName, @@ -57,6 +58,7 @@ namespace edm { //void endFile(); void writeLuminosityBlock(LuminosityBlockForOutput const& lb); void writeRun(RunForOutput const& r); + void writeProcessBlock(ProcessBlockForOutput const&); void writeFileFormatVersion(); void writeFileIdentifier(); void writeIndexIntoFile(); @@ -69,6 +71,7 @@ namespace edm { void writeThinnedAssociationsHelper(); void writeProductDependencies(); void writeEventAuxiliary(); + void writeProcessBlockHelper(); void finishEndFile(); void beginInputFile(FileBlock const& fb, int remainingEvents); @@ -78,17 +81,11 @@ namespace edm { std::string const& fileName() const { return file_; } private: - //------------------------------- - // Local types - // - - //------------------------------- - // Private functions - - void setBranchAliases(TTree* tree, SelectedProducts const& branches) const; + void setBranchAliases(TTree* tree, SelectedProducts const& branches, std::string const& processName) const; void fillBranches(BranchType const& branchType, OccurrenceForOutput const& occurrence, + unsigned int ttreeIndex, StoredProductProvenanceVector* productProvenanceVecPtr = nullptr, ProductProvenanceRetriever const* provRetriever = nullptr); @@ -135,11 +132,13 @@ namespace edm { StoredProductProvenanceVector eventEntryInfoVector_; edm::propagate_const pEventEntryInfoVector_; BranchListIndexes const* pBranchListIndexes_; + EventToProcessBlockIndexes const* pEventToProcessBlockIndexes_; EventSelectionIDVector const* pEventSelectionIDs_; RootOutputTree eventTree_; RootOutputTree lumiTree_; RootOutputTree runTree_; - RootOutputTreePtrArray treePointers_; + std::vector>> processBlockTrees_; + std::vector> treePointers_; bool dataTypeReported_; ProcessHistoryRegistry processHistoryRegistry_; std::map parentageIDs_; diff --git a/IOPool/Output/src/RootOutputTree.cc b/IOPool/Output/src/RootOutputTree.cc index f53550cc34511..c86b9712ed2d2 100644 --- a/IOPool/Output/src/RootOutputTree.cc +++ b/IOPool/Output/src/RootOutputTree.cc @@ -96,9 +96,12 @@ namespace edm { RootOutputTree::RootOutputTree(std::shared_ptr filePtr, BranchType const& branchType, int splitLevel, - int treeMaxVirtualSize) + int treeMaxVirtualSize, + std::string const& processName) : filePtr_(filePtr), - tree_(makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType), splitLevel)), + tree_(processName.empty() + ? makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType), splitLevel) + : makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType, processName), splitLevel)), producedBranches_(), readBranches_(), auxBranches_(), diff --git a/IOPool/Output/src/RootOutputTree.h b/IOPool/Output/src/RootOutputTree.h index 8c298be2c4287..ea582e2a7e3f9 100644 --- a/IOPool/Output/src/RootOutputTree.h +++ b/IOPool/Output/src/RootOutputTree.h @@ -23,7 +23,11 @@ class TBranch; namespace edm { class RootOutputTree { public: - RootOutputTree(std::shared_ptr filePtr, BranchType const& branchType, int splitLevel, int treeMaxVirtualSize); + RootOutputTree(std::shared_ptr filePtr, + BranchType const& branchType, + int splitLevel, + int treeMaxVirtualSize, + std::string const& processName = std::string()); ~RootOutputTree() {} diff --git a/Mixing/Base/src/SecondaryEventProvider.cc b/Mixing/Base/src/SecondaryEventProvider.cc index a220c1065f0ee..589e3d1fdb181 100644 --- a/Mixing/Base/src/SecondaryEventProvider.cc +++ b/Mixing/Base/src/SecondaryEventProvider.cc @@ -1,5 +1,6 @@ #include "FWCore/Concurrency/interface/include_first_syncWait.h" #include "Mixing/Base/src/SecondaryEventProvider.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" #include "FWCore/Framework/interface/ExceptionActions.h" #include "FWCore/Framework/src/PreallocationConfiguration.h" #include "FWCore/Framework/src/TransitionInfoTypes.h" @@ -67,7 +68,8 @@ namespace edm { void SecondaryEventProvider::beginJob(ProductRegistry const& iRegistry, eventsetup::ESRecordsToProxyIndices const& iIndices) { - workerManager_.beginJob(iRegistry, iIndices); + ProcessBlockHelper dummyProcessBlockHelper; + workerManager_.beginJob(iRegistry, iIndices, dummyProcessBlockHelper); } //NOTE: When the Stream interfaces are propagated to the modules, this code must be updated From 4469486d71f0446d5d85147802a747e82ee65b5a Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 00:19:01 +0200 Subject: [PATCH 02/26] Minor style improvements, PR comments --- DataFormats/Provenance/src/StoredProcessBlockHelper.cc | 2 +- DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DataFormats/Provenance/src/StoredProcessBlockHelper.cc b/DataFormats/Provenance/src/StoredProcessBlockHelper.cc index 2814eb2e2c5de..2f7d6daed5e34 100644 --- a/DataFormats/Provenance/src/StoredProcessBlockHelper.cc +++ b/DataFormats/Provenance/src/StoredProcessBlockHelper.cc @@ -2,7 +2,7 @@ namespace edm { - StoredProcessBlockHelper::StoredProcessBlockHelper() {} + StoredProcessBlockHelper::StoredProcessBlockHelper() = default; StoredProcessBlockHelper::StoredProcessBlockHelper(std::vector const& processesWithProcessBlockProducts) : processesWithProcessBlockProducts_(processesWithProcessBlockProducts) {} diff --git a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp index 504a0372f7b4e..144fae65aadb2 100644 --- a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp +++ b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp @@ -28,7 +28,7 @@ TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { } SECTION("Constructor") { - std::vector testStrings{std::string("test1"), std::string("test2"), std::string("test3")}; + std::vector testStrings{"test1", "test2", "test3"}; edm::StoredProcessBlockHelper storedProcessBlockHelper(testStrings); REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts() == testStrings); From baad1a6cb7cf29f5afe79b1b8f527406345b3a83 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 19:20:35 +0200 Subject: [PATCH 03/26] Remove unnecessary duplicate functions invalidCacheIndex() and invalidProcessIndex() --- DataFormats/Provenance/interface/StoredProcessBlockHelper.h | 3 --- DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp | 3 --- FWCore/Common/interface/OutputProcessBlockHelper.h | 2 -- FWCore/Common/src/OutputProcessBlockHelper.cc | 4 ++-- FWCore/Common/src/ProcessBlockHelper.cc | 4 ++-- FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc | 2 -- 6 files changed, 4 insertions(+), 14 deletions(-) diff --git a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h index af906608ea4b1..3af272ff56509 100644 --- a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h +++ b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h @@ -38,9 +38,6 @@ namespace edm { std::vector& processBlockCacheIndices() { return processBlockCacheIndices_; } - static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; } - static constexpr unsigned int invalidProcessIndex() { return 0xffffffff; } - private: std::vector processesWithProcessBlockProducts_; diff --git a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp index 144fae65aadb2..a878cc3bd6935 100644 --- a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp +++ b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp @@ -18,9 +18,6 @@ TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { REQUIRE(storedProcessBlockHelperConstPtr->processesWithProcessBlockProducts().empty()); REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); - REQUIRE(edm::StoredProcessBlockHelper::invalidCacheIndex() == 0xffffffff); - REQUIRE(edm::StoredProcessBlockHelper::invalidProcessIndex() == 0xffffffff); - edm::EventToProcessBlockIndexes eventToProcessBlockIndexes; REQUIRE(eventToProcessBlockIndexes.index() == 0); eventToProcessBlockIndexes.setIndex(2); diff --git a/FWCore/Common/interface/OutputProcessBlockHelper.h b/FWCore/Common/interface/OutputProcessBlockHelper.h index 7ee5bd413cd50..5bc7064067620 100644 --- a/FWCore/Common/interface/OutputProcessBlockHelper.h +++ b/FWCore/Common/interface/OutputProcessBlockHelper.h @@ -30,8 +30,6 @@ namespace edm { bool productsFromInputKept() const { return productsFromInputKept_; } - static constexpr unsigned int invalidCacheIndex() { return 0xffffffff; } - ProcessBlockHelperBase const* processBlockHelper() const { return processBlockHelper_; } // These are intended to be used only for testing purposes diff --git a/FWCore/Common/src/OutputProcessBlockHelper.cc b/FWCore/Common/src/OutputProcessBlockHelper.cc index 70de57d56e5fd..7888bd1cc5798 100644 --- a/FWCore/Common/src/OutputProcessBlockHelper.cc +++ b/FWCore/Common/src/OutputProcessBlockHelper.cc @@ -162,10 +162,10 @@ namespace edm { // Really fill the cache indices that will be stored in the output file in this loop for (unsigned int iStoredProcess = 0; iStoredProcess < nStoredProcesses; ++iStoredProcess) { - unsigned int storedCacheIndex = invalidCacheIndex(); + unsigned int storedCacheIndex = ProcessBlockHelperBase::invalidCacheIndex(); if (iStoredProcess < nInputProcesses) { unsigned int cacheIndex = innerVectorOfCacheIndices[translateFromStoredIndex_[iStoredProcess]]; - if (cacheIndex != invalidCacheIndex()) { + if (cacheIndex != ProcessBlockHelperBase::invalidCacheIndex()) { // The offsets count in the cache sequence to the first entry in // the current input file and process unsigned int inputOffset = fileOffset + processOffset[iStoredProcess]; diff --git a/FWCore/Common/src/ProcessBlockHelper.cc b/FWCore/Common/src/ProcessBlockHelper.cc index 56e14a71e9c30..597865cedc962 100644 --- a/FWCore/Common/src/ProcessBlockHelper.cc +++ b/FWCore/Common/src/ProcessBlockHelper.cc @@ -206,8 +206,8 @@ namespace edm { for (unsigned int j = 0; j < nFinalProcesses; ++j) { unsigned int storedIndex = finalIndexToStoredIndex[j]; unsigned int oldCacheIndex = storedCacheIndices[storedOuterOffset + storedIndex]; - unsigned int newCacheIndex = StoredProcessBlockHelper::invalidCacheIndex(); - if (oldCacheIndex != StoredProcessBlockHelper::invalidCacheIndex()) { + unsigned int newCacheIndex = invalidCacheIndex(); + if (oldCacheIndex != invalidCacheIndex()) { newCacheIndex = oldCacheIndex - oldOffsets[storedIndex] + newOffsets[j]; } newCacheIndices.emplace_back(newCacheIndex); diff --git a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc index 483d35bf38511..c7f5decc2c959 100644 --- a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc +++ b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc @@ -13,8 +13,6 @@ TEST_CASE("Test ProcessBlockHelpers", "[ProcessBlockHelpers]") { SECTION("OutputProcessBlockHelper") { edm::OutputProcessBlockHelper outputProcessBlockHelper; - - REQUIRE(edm::OutputProcessBlockHelper::invalidCacheIndex() == 0xffffffff); } SECTION("ProcessBlockHelper") { From 67083514f9d3afe3d2d0264edc253da87164d805 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 19:36:38 +0200 Subject: [PATCH 04/26] Move virtual function to source file ~ProcessBlockHelperBase() --- FWCore/Common/interface/ProcessBlockHelperBase.h | 2 +- FWCore/Common/src/ProcessBlockHelperBase.cc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/FWCore/Common/interface/ProcessBlockHelperBase.h b/FWCore/Common/interface/ProcessBlockHelperBase.h index ba98b9f40a5e3..1ffd4e30be832 100644 --- a/FWCore/Common/interface/ProcessBlockHelperBase.h +++ b/FWCore/Common/interface/ProcessBlockHelperBase.h @@ -17,7 +17,7 @@ namespace edm { class ProcessBlockHelperBase { public: - virtual ~ProcessBlockHelperBase() = default; + virtual ~ProcessBlockHelperBase(); std::vector& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; } std::vector const& processesWithProcessBlockProducts() const { diff --git a/FWCore/Common/src/ProcessBlockHelperBase.cc b/FWCore/Common/src/ProcessBlockHelperBase.cc index 0d123adefe53a..cd692af54aeef 100644 --- a/FWCore/Common/src/ProcessBlockHelperBase.cc +++ b/FWCore/Common/src/ProcessBlockHelperBase.cc @@ -8,6 +8,8 @@ namespace edm { + ProcessBlockHelperBase::~ProcessBlockHelperBase() = default; + void ProcessBlockHelperBase::updateForNewProcess(ProductRegistry const& productRegistry, std::string const& processName) { // Add the current process at the end if there are any From 8cb78de626079e531fa34ae0ab364b2427538d6f Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 21:13:51 +0200 Subject: [PATCH 05/26] Replace for loop which std::find_if --- FWCore/Common/src/ProcessBlockHelperBase.cc | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/FWCore/Common/src/ProcessBlockHelperBase.cc b/FWCore/Common/src/ProcessBlockHelperBase.cc index cd692af54aeef..20f903525b37c 100644 --- a/FWCore/Common/src/ProcessBlockHelperBase.cc +++ b/FWCore/Common/src/ProcessBlockHelperBase.cc @@ -6,6 +6,9 @@ #include "FWCore/Utilities/interface/ProductLabels.h" #include "FWCore/Utilities/interface/TypeID.h" +#include +#include + namespace edm { ProcessBlockHelperBase::~ProcessBlockHelperBase() = default; @@ -37,18 +40,17 @@ namespace edm { desc.moduleLabel() == productLabels.module && desc.productInstanceName() == productLabels.productInstance && desc.unwrappedTypeID() == typeID && (processName.empty() || processName == desc.processName())) { // This code is to select the latest matching process - unsigned int position = 0; - bool found = false; - for (auto const& processFromHelper : processesWithProcessBlockProducts_) { - if (processFromHelper == desc.processName()) { - found = true; - break; + auto found = std::find_if(processesWithProcessBlockProducts_.begin(), + processesWithProcessBlockProducts_.end(), + [&desc](auto const& processFromHelper) { + return processFromHelper == desc.processName(); + }); + if (found != processesWithProcessBlockProducts_.end()) { + const unsigned int position = std::distance(processesWithProcessBlockProducts_.begin(), found); + if (position >= bestPosition) { + bestPosition = position; + selectedProcess = desc.processName(); } - ++position; - } - if (found && position >= bestPosition) { - bestPosition = position; - selectedProcess = desc.processName(); } } } From f59a1132ec743ec107ac3f1880c12368794a19f1 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 23:11:23 +0200 Subject: [PATCH 06/26] Replace non-const accessors --- FWCore/Common/interface/ProcessBlockHelperBase.h | 6 ++++-- FWCore/Common/src/ProcessBlockHelper.cc | 2 +- FWCore/Common/src/SubProcessBlockHelper.cc | 4 ++-- .../test/test_catch2_ProcessBlockHelpers.cc | 16 +++++++++++----- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/FWCore/Common/interface/ProcessBlockHelperBase.h b/FWCore/Common/interface/ProcessBlockHelperBase.h index 1ffd4e30be832..1aa33917128ab 100644 --- a/FWCore/Common/interface/ProcessBlockHelperBase.h +++ b/FWCore/Common/interface/ProcessBlockHelperBase.h @@ -19,13 +19,15 @@ namespace edm { public: virtual ~ProcessBlockHelperBase(); - std::vector& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; } std::vector const& processesWithProcessBlockProducts() const { return processesWithProcessBlockProducts_; } + void setProcessesWithProcessBlockProducts(std::vector const& val) { processesWithProcessBlockProducts_ = val; } + void emplaceBackProcessName(std::string const& processName) { processesWithProcessBlockProducts_.emplace_back(processName); } - std::vector& addedProcesses() { return addedProcesses_; } std::vector const& addedProcesses() const { return addedProcesses_; } + void setAddedProcesses(std::vector const& val) { addedProcesses_ = val; } + void emplaceBackAddedProcessName(std::string const& processName) { addedProcesses_.emplace_back(processName); } void updateForNewProcess(ProductRegistry const&, std::string const& processName); diff --git a/FWCore/Common/src/ProcessBlockHelper.cc b/FWCore/Common/src/ProcessBlockHelper.cc index 597865cedc962..80b669165f99a 100644 --- a/FWCore/Common/src/ProcessBlockHelper.cc +++ b/FWCore/Common/src/ProcessBlockHelper.cc @@ -124,7 +124,7 @@ namespace edm { initializedFromInput_ = true; assert(processesWithProcessBlockProducts().empty()); - processesWithProcessBlockProducts() = storedProcessBlockHelper.processesWithProcessBlockProducts(); + setProcessesWithProcessBlockProducts(storedProcessBlockHelper.processesWithProcessBlockProducts()); nProcessesInFirstFile_ = processesWithProcessBlockProducts().size(); } } diff --git a/FWCore/Common/src/SubProcessBlockHelper.cc b/FWCore/Common/src/SubProcessBlockHelper.cc index 157808e316201..cc05fc4334fe5 100644 --- a/FWCore/Common/src/SubProcessBlockHelper.cc +++ b/FWCore/Common/src/SubProcessBlockHelper.cc @@ -53,7 +53,7 @@ namespace edm { for (auto const& item : productRegistry.productList()) { BranchDescription const& desc = item.second; if (desc.branchType() == InProcess && desc.present() && desc.processName() == processName) { - processesWithProcessBlockProducts().emplace_back(processName); + emplaceBackProcessName(processName); break; } } @@ -65,7 +65,7 @@ namespace edm { for (auto const& item : productRegistry.productList()) { BranchDescription const& desc = item.second; if (desc.branchType() == InProcess && desc.present() && desc.processName() == processName) { - addedProcesses().emplace_back(processName); + emplaceBackAddedProcessName(processName); break; } } diff --git a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc index c7f5decc2c959..3618671a36616 100644 --- a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc +++ b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc @@ -8,6 +8,7 @@ #include TEST_CASE("Test ProcessBlockHelpers", "[ProcessBlockHelpers]") { + const std::string testString("ADD"); const std::vector testNames = {{"HLT"}, {"RECO"}, {"TEST"}}; const std::vector testNames2 = {{"MERGE"}, {"ANA"}, {"HARVEST"}}; @@ -17,14 +18,19 @@ TEST_CASE("Test ProcessBlockHelpers", "[ProcessBlockHelpers]") { SECTION("ProcessBlockHelper") { edm::ProcessBlockHelper processBlockHelper; - processBlockHelper.processesWithProcessBlockProducts() = testNames; - edm::ProcessBlockHelper const& constProcessBlockHelper = processBlockHelper; + processBlockHelper.setProcessesWithProcessBlockProducts(testNames); REQUIRE(processBlockHelper.processesWithProcessBlockProducts() == testNames); - REQUIRE(constProcessBlockHelper.processesWithProcessBlockProducts() == testNames); + processBlockHelper.emplaceBackProcessName(testString); + std::vector testEmplace = testNames; + testEmplace.emplace_back(testString); + REQUIRE(processBlockHelper.processesWithProcessBlockProducts() == testEmplace); - processBlockHelper.addedProcesses() = testNames2; + processBlockHelper.setAddedProcesses(testNames2); REQUIRE(processBlockHelper.addedProcesses() == testNames2); - REQUIRE(constProcessBlockHelper.addedProcesses() == testNames2); + processBlockHelper.emplaceBackAddedProcessName(testString); + std::vector testEmplace2 = testNames2; + testEmplace2.emplace_back(testString); + REQUIRE(processBlockHelper.addedProcesses() == testEmplace2); REQUIRE(edm::ProcessBlockHelper::invalidCacheIndex() == 0xffffffff); REQUIRE(edm::ProcessBlockHelper::invalidProcessIndex() == 0xffffffff); From 44b510763064021ea35e737cbcdbd79ec93c4850 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 17 Jun 2021 23:47:38 +0200 Subject: [PATCH 07/26] Make interface more flexible Don't require FileBlock to have rvalue reference for its constructor and update functions --- FWCore/Framework/interface/FileBlock.h | 8 ++++---- FWCore/Framework/src/FileBlock.cc | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/FWCore/Framework/interface/FileBlock.h b/FWCore/Framework/interface/FileBlock.h index b4a4a91631eda..bdcaa65aefed4 100644 --- a/FWCore/Framework/interface/FileBlock.h +++ b/FWCore/Framework/interface/FileBlock.h @@ -78,8 +78,8 @@ namespace edm { TTree* lumiMeta, TTree* run, TTree* runMeta, - std::vector&& processBlockTrees, - std::vector&& processesWithProcessBlockTrees, + std::vector processBlockTrees, + std::vector processesWithProcessBlockTrees, int whyNotFastClonable, std::array const& hasNewlyDroppedBranch, std::string const& fileName, @@ -110,8 +110,8 @@ namespace edm { TTree* lumiMeta, TTree* run, TTree* runMeta, - std::vector&& processBlockTrees, - std::vector&& processesWithProcessBlockTrees); + std::vector processBlockTrees, + std::vector processesWithProcessBlockTrees); FileFormatVersion const& fileFormatVersion() const { return fileFormatVersion_; } TTree* tree() const { return tree_; } diff --git a/FWCore/Framework/src/FileBlock.cc b/FWCore/Framework/src/FileBlock.cc index 8fbea961326db..f930263812a77 100644 --- a/FWCore/Framework/src/FileBlock.cc +++ b/FWCore/Framework/src/FileBlock.cc @@ -9,8 +9,8 @@ namespace edm { TTree* lumiMeta, TTree* run, TTree* runMeta, - std::vector&& processBlockTrees, - std::vector&& processesWithProcessBlockTrees) { + std::vector processBlockTrees, + std::vector processesWithProcessBlockTrees) { tree_ = ev; metaTree_ = meta; lumiTree_ = lumi; From 3625c410a75bf96c24d692ae9a23b2d39541fc54 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 18 Jun 2021 00:07:07 +0200 Subject: [PATCH 08/26] Remove unnecessary line --- FWCore/Framework/interface/InputProcessBlockCacheImpl.h | 1 - 1 file changed, 1 deletion(-) diff --git a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h index d9737af39b0f9..47ba8c56f7ca4 100644 --- a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h +++ b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h @@ -63,7 +63,6 @@ namespace edm { } struct TokenInfo { - public: EDGetToken token_; TypeID typeID_; }; From e2b4a3ad45fdac9a8b76698ea7692b2a1419198b Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 18 Jun 2021 21:12:41 +0200 Subject: [PATCH 09/26] Rename function to moveProcessBlockCacheFiller --- FWCore/Framework/interface/InputProcessBlockCacheImpl.h | 7 +++++-- FWCore/Framework/interface/stream/callAbilities.h | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h index 47ba8c56f7ca4..6979978859c41 100644 --- a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h +++ b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h @@ -148,8 +148,11 @@ namespace edm { registerProcessBlockCacheFiller(token, std::forward(func)); } - // This gets used for stream type modules instead of registerProcessBlockCacheFiller - void copyProcessBlockCacheFiller(std::vector& tokenInfos, + // This gets used for stream type modules where the InputProcessBlockCacheImpl + // object is held by the adaptor. For stream modules, we use a registerProcessBlockCacheFiller + // function defined in edm::stream::impl::InputProcessBlockCacheHolder then + // move the information. + void moveProcessBlockCacheFiller(std::vector& tokenInfos, std::tuple...>& functors) { tokenInfos_ = std::move(tokenInfos); functors_ = std::move(functors); diff --git a/FWCore/Framework/interface/stream/callAbilities.h b/FWCore/Framework/interface/stream/callAbilities.h index df20d8c94d71f..5c231b665cb51 100644 --- a/FWCore/Framework/interface/stream/callAbilities.h +++ b/FWCore/Framework/interface/stream/callAbilities.h @@ -78,7 +78,7 @@ namespace edm { unsigned int iStreamModule) { iProd->setProcessBlockCache(iCaches->get()); if (iStreamModule == 0 && iProd->cacheFillersRegistered()) { - (*iCaches)->copyProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); + (*iCaches)->moveProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); } iProd->clearRegistration(); } @@ -111,7 +111,7 @@ namespace edm { unsigned int iStreamModule) { iProd->setProcessBlockCache(iCaches->get()); if (iStreamModule == 0 && iProd->cacheFillersRegistered()) { - (*iCaches)->copyProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); + (*iCaches)->moveProcessBlockCacheFiller(iProd->tokenInfos(), iProd->cacheFillers()); } iProd->clearRegistration(); } From 2202de04ff17d6bcb5bca475b69c5c7285cde121 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 18 Jun 2021 21:30:32 +0200 Subject: [PATCH 10/26] Use implicit conversion, more readable --- FWCore/Framework/interface/InputProcessBlockCacheImpl.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h index 6979978859c41..d21f50b44d158 100644 --- a/FWCore/Framework/interface/InputProcessBlockCacheImpl.h +++ b/FWCore/Framework/interface/InputProcessBlockCacheImpl.h @@ -215,9 +215,7 @@ namespace edm { tokenInfos_[ICacheType] = TokenInfo{EDGetToken(token), TypeID(typeid(DataType))}; - std::get(functors_).func_ = - std::function(ProcessBlock const&, std::shared_ptr const&)>( - std::forward(func)); + std::get(functors_).func_ = std::forward(func); } // ------------ Data members -------------------- From e92d6af641d093134416f13fd9e2fb8c0d814338 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Sat, 19 Jun 2021 00:38:46 +0200 Subject: [PATCH 11/26] Make constructor explicit --- DataFormats/Provenance/interface/StoredProcessBlockHelper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h index 3af272ff56509..a850b5656d75a 100644 --- a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h +++ b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h @@ -26,7 +26,7 @@ namespace edm { // This constructor exists for ROOT I/O StoredProcessBlockHelper(); - StoredProcessBlockHelper(std::vector const& processesWithProcessBlockProducts); + explicit StoredProcessBlockHelper(std::vector const& processesWithProcessBlockProducts); std::vector const& processesWithProcessBlockProducts() const { return processesWithProcessBlockProducts_; From a49422971888d43a7d0f4cbb53f77e674970fb79 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 24 Jun 2021 22:05:09 +0200 Subject: [PATCH 12/26] code format --- FWCore/Common/interface/ProcessBlockHelperBase.h | 8 ++++++-- FWCore/Common/src/ProcessBlockHelperBase.cc | 9 ++++----- FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc | 4 +--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/FWCore/Common/interface/ProcessBlockHelperBase.h b/FWCore/Common/interface/ProcessBlockHelperBase.h index 1aa33917128ab..dd51001397319 100644 --- a/FWCore/Common/interface/ProcessBlockHelperBase.h +++ b/FWCore/Common/interface/ProcessBlockHelperBase.h @@ -22,8 +22,12 @@ namespace edm { std::vector const& processesWithProcessBlockProducts() const { return processesWithProcessBlockProducts_; } - void setProcessesWithProcessBlockProducts(std::vector const& val) { processesWithProcessBlockProducts_ = val; } - void emplaceBackProcessName(std::string const& processName) { processesWithProcessBlockProducts_.emplace_back(processName); } + void setProcessesWithProcessBlockProducts(std::vector const& val) { + processesWithProcessBlockProducts_ = val; + } + void emplaceBackProcessName(std::string const& processName) { + processesWithProcessBlockProducts_.emplace_back(processName); + } std::vector const& addedProcesses() const { return addedProcesses_; } void setAddedProcesses(std::vector const& val) { addedProcesses_ = val; } diff --git a/FWCore/Common/src/ProcessBlockHelperBase.cc b/FWCore/Common/src/ProcessBlockHelperBase.cc index 20f903525b37c..25b6f93b28d38 100644 --- a/FWCore/Common/src/ProcessBlockHelperBase.cc +++ b/FWCore/Common/src/ProcessBlockHelperBase.cc @@ -40,11 +40,10 @@ namespace edm { desc.moduleLabel() == productLabels.module && desc.productInstanceName() == productLabels.productInstance && desc.unwrappedTypeID() == typeID && (processName.empty() || processName == desc.processName())) { // This code is to select the latest matching process - auto found = std::find_if(processesWithProcessBlockProducts_.begin(), - processesWithProcessBlockProducts_.end(), - [&desc](auto const& processFromHelper) { - return processFromHelper == desc.processName(); - }); + auto found = + std::find_if(processesWithProcessBlockProducts_.begin(), + processesWithProcessBlockProducts_.end(), + [&desc](auto const& processFromHelper) { return processFromHelper == desc.processName(); }); if (found != processesWithProcessBlockProducts_.end()) { const unsigned int position = std::distance(processesWithProcessBlockProducts_.begin(), found); if (position >= bestPosition) { diff --git a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc index 3618671a36616..8f2f2225f7adf 100644 --- a/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc +++ b/FWCore/Common/test/test_catch2_ProcessBlockHelpers.cc @@ -12,9 +12,7 @@ TEST_CASE("Test ProcessBlockHelpers", "[ProcessBlockHelpers]") { const std::vector testNames = {{"HLT"}, {"RECO"}, {"TEST"}}; const std::vector testNames2 = {{"MERGE"}, {"ANA"}, {"HARVEST"}}; - SECTION("OutputProcessBlockHelper") { - edm::OutputProcessBlockHelper outputProcessBlockHelper; - } + SECTION("OutputProcessBlockHelper") { edm::OutputProcessBlockHelper outputProcessBlockHelper; } SECTION("ProcessBlockHelper") { edm::ProcessBlockHelper processBlockHelper; From f61791a40fc31ad24ec45df32abf7402fa7b0deb Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 24 Jun 2021 22:06:11 +0200 Subject: [PATCH 13/26] Use set functions instead of non-const accessor in StoredProcessBlockHelper --- .../interface/StoredProcessBlockHelper.h | 9 ++++--- .../test/StoredProcessBlockHelper_t.cpp | 2 +- FWCore/Common/interface/ProcessBlockHelper.h | 9 +++---- FWCore/Common/src/OutputProcessBlockHelper.cc | 5 +++- FWCore/Common/src/ProcessBlockHelper.cc | 26 ++++++++++--------- IOPool/Input/src/RootFile.cc | 19 +++++--------- 6 files changed, 34 insertions(+), 36 deletions(-) diff --git a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h index a850b5656d75a..bae93e4e43738 100644 --- a/DataFormats/Provenance/interface/StoredProcessBlockHelper.h +++ b/DataFormats/Provenance/interface/StoredProcessBlockHelper.h @@ -17,6 +17,7 @@ interact with the ProcessBlockHelper. */ #include +#include #include namespace edm { @@ -31,12 +32,12 @@ namespace edm { std::vector const& processesWithProcessBlockProducts() const { return processesWithProcessBlockProducts_; } - - std::vector& processesWithProcessBlockProducts() { return processesWithProcessBlockProducts_; } + void setProcessesWithProcessBlockProducts(std::vector val) { + processesWithProcessBlockProducts_ = std::move(val); + } std::vector const& processBlockCacheIndices() const { return processBlockCacheIndices_; } - - std::vector& processBlockCacheIndices() { return processBlockCacheIndices_; } + void setProcessBlockCacheIndices(std::vector val) { processBlockCacheIndices_ = std::move(val); } private: std::vector processesWithProcessBlockProducts_; diff --git a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp index a878cc3bd6935..4ba3f7aaa1406 100644 --- a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp +++ b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp @@ -36,7 +36,7 @@ TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); std::vector testIndices{1, 10, 100}; - storedProcessBlockHelper.processBlockCacheIndices() = testIndices; + storedProcessBlockHelper.setProcessBlockCacheIndices(testIndices); REQUIRE(storedProcessBlockHelper.processBlockCacheIndices() == testIndices); REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices() == testIndices); } diff --git a/FWCore/Common/interface/ProcessBlockHelper.h b/FWCore/Common/interface/ProcessBlockHelper.h index 176dc6110faca..bce4a1c729ed7 100644 --- a/FWCore/Common/interface/ProcessBlockHelper.h +++ b/FWCore/Common/interface/ProcessBlockHelper.h @@ -30,14 +30,12 @@ namespace edm { bool initializedFromInput() const { return initializedFromInput_; } - bool firstFileDropProcessesAndReorderStored(std::vector& storedProcesses, - std::vector& storedCacheIndices, + bool firstFileDropProcessesAndReorderStored(StoredProcessBlockHelper& storedProcessBlockHelper, std::set const& processesToKeep, std::vector const& nEntries, std::vector& finalIndexToStoredIndex) const; - bool dropProcessesAndReorderStored(std::vector& storedProcesses, - std::vector& storedCacheIndices, + bool dropProcessesAndReorderStored(StoredProcessBlockHelper& storedProcessBlockHelper, std::set const& processesToKeep, std::vector const& nEntries, std::vector& finalIndexToStoredIndex, @@ -51,8 +49,7 @@ namespace edm { void clearAfterOutputFilesClose(); private: - void dropProcessesAndReorderStoredImpl(std::vector& storedProcesses, - std::vector& storedCacheIndices, + void dropProcessesAndReorderStoredImpl(StoredProcessBlockHelper& storedProcessBlockHelper, std::vector& finalProcesses, std::vector const& nEntries, std::vector const& finalIndexToStoredIndex) const; diff --git a/FWCore/Common/src/OutputProcessBlockHelper.cc b/FWCore/Common/src/OutputProcessBlockHelper.cc index 7888bd1cc5798..0e695d51de4ae 100644 --- a/FWCore/Common/src/OutputProcessBlockHelper.cc +++ b/FWCore/Common/src/OutputProcessBlockHelper.cc @@ -5,6 +5,7 @@ #include #include +#include namespace edm { @@ -54,7 +55,7 @@ namespace edm { // The stored cache indices are the ones we want to fill. // This will get written to the output file. // Note for output the vector of vectors is flattened into a single vector - std::vector& storedCacheIndices = storedProcessBlockHelper.processBlockCacheIndices(); + std::vector storedCacheIndices; // Number of processes in StoredProcessBlockHelper. unsigned int nStoredProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts().size(); @@ -68,6 +69,7 @@ namespace edm { for (unsigned int i = 0; i < nStoredProcesses; ++i) { storedCacheIndices.push_back(i); } + storedProcessBlockHelper.setProcessBlockCacheIndices(std::move(storedCacheIndices)); return; } @@ -180,6 +182,7 @@ namespace edm { storedCacheIndices.push_back(storedCacheIndex); } } + storedProcessBlockHelper.setProcessBlockCacheIndices(std::move(storedCacheIndices)); } void OutputProcessBlockHelper::setStoredProcessOffset(unsigned int nInputProcesses, diff --git a/FWCore/Common/src/ProcessBlockHelper.cc b/FWCore/Common/src/ProcessBlockHelper.cc index 80b669165f99a..7f0d65473d685 100644 --- a/FWCore/Common/src/ProcessBlockHelper.cc +++ b/FWCore/Common/src/ProcessBlockHelper.cc @@ -45,11 +45,12 @@ namespace edm { // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks // to be read in the same order for all input files. bool ProcessBlockHelper::firstFileDropProcessesAndReorderStored( - std::vector& storedProcesses, - std::vector& storedCacheIndices, + StoredProcessBlockHelper& storedProcessBlockHelper, std::set const& processesToKeep, std::vector const& nEntries, std::vector& finalIndexToStoredIndex) const { + std::vector const& storedProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts(); + bool isModified = false; unsigned int finalSize = 0; for (auto const& process : storedProcesses) { @@ -72,20 +73,20 @@ namespace edm { finalIndexToStoredIndex.emplace_back(iStored); } } - dropProcessesAndReorderStoredImpl( - storedProcesses, storedCacheIndices, finalProcesses, nEntries, finalIndexToStoredIndex); + dropProcessesAndReorderStoredImpl(storedProcessBlockHelper, finalProcesses, nEntries, finalIndexToStoredIndex); return isModified; } // Modifies the process names and cache indices in the StoredProcessBlockHelper. // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks // to be read in the same order for all input files. - bool ProcessBlockHelper::dropProcessesAndReorderStored(std::vector& storedProcesses, - std::vector& storedCacheIndices, + bool ProcessBlockHelper::dropProcessesAndReorderStored(StoredProcessBlockHelper& storedProcessBlockHelper, std::set const& processesToKeep, std::vector const& nEntries, std::vector& finalIndexToStoredIndex, std::vector const& firstFileFinalProcesses) const { + std::vector const& storedProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts(); + std::vector finalProcesses; // Always will allocate enough space (sometimes too much) finalProcesses.reserve(storedProcesses.size()); @@ -114,8 +115,7 @@ namespace edm { return isModified; } - dropProcessesAndReorderStoredImpl( - storedProcesses, storedCacheIndices, finalProcesses, nEntries, finalIndexToStoredIndex); + dropProcessesAndReorderStoredImpl(storedProcessBlockHelper, finalProcesses, nEntries, finalIndexToStoredIndex); return isModified; } @@ -168,11 +168,13 @@ namespace edm { // Part of the dropOnInput. Also part of reordering that forces ProcessBlocks // to be read in the same order for all input files. void ProcessBlockHelper::dropProcessesAndReorderStoredImpl( - std::vector& storedProcesses, - std::vector& storedCacheIndices, + StoredProcessBlockHelper& storedProcessBlockHelper, std::vector& finalProcesses, std::vector const& nEntries, std::vector const& finalIndexToStoredIndex) const { + std::vector const& storedProcesses = storedProcessBlockHelper.processesWithProcessBlockProducts(); + std::vector const& storedCacheIndices = storedProcessBlockHelper.processBlockCacheIndices(); + assert(!storedProcesses.empty()); std::vector newCacheIndices; if (!finalProcesses.empty()) { @@ -215,8 +217,8 @@ namespace edm { storedOuterOffset += storedProcesses.size(); } } - storedCacheIndices.swap(newCacheIndices); - storedProcesses.swap(finalProcesses); + storedProcessBlockHelper.setProcessBlockCacheIndices(std::move(newCacheIndices)); + storedProcessBlockHelper.setProcessesWithProcessBlockProducts(std::move(finalProcesses)); } void ProcessBlockHelper::fillFromPrimaryInputWhenNotEmpty(std::vector const& storedProcesses, diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index d44daf63e5fb0..75c0587357a81 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -2068,19 +2068,14 @@ namespace edm { if (firstInputFile) { isModified = processBlockHelper->firstFileDropProcessesAndReorderStored( - storedProcessBlockHelper.processesWithProcessBlockProducts(), - storedProcessBlockHelper.processBlockCacheIndices(), - processesWithKeptProcessBlockProducts, - nEntries, - finalIndexToStoredIndex); + storedProcessBlockHelper, processesWithKeptProcessBlockProducts, nEntries, finalIndexToStoredIndex); } else { - isModified = processBlockHelper->dropProcessesAndReorderStored( - storedProcessBlockHelper.processesWithProcessBlockProducts(), - storedProcessBlockHelper.processBlockCacheIndices(), - processesWithKeptProcessBlockProducts, - nEntries, - finalIndexToStoredIndex, - processBlockHelper->processesWithProcessBlockProducts()); + isModified = + processBlockHelper->dropProcessesAndReorderStored(storedProcessBlockHelper, + processesWithKeptProcessBlockProducts, + nEntries, + finalIndexToStoredIndex, + processBlockHelper->processesWithProcessBlockProducts()); } // At this point, any modifications to storedProcessBlockHelper are done. From a06c464ff2d1cab10e8e28d62266441ba134a815 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 24 Jun 2021 22:55:09 +0200 Subject: [PATCH 14/26] Make functions private and test module a friend --- FWCore/Common/interface/OutputProcessBlockHelper.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/FWCore/Common/interface/OutputProcessBlockHelper.h b/FWCore/Common/interface/OutputProcessBlockHelper.h index 5bc7064067620..6a09291c1d916 100644 --- a/FWCore/Common/interface/OutputProcessBlockHelper.h +++ b/FWCore/Common/interface/OutputProcessBlockHelper.h @@ -32,11 +32,12 @@ namespace edm { ProcessBlockHelperBase const* processBlockHelper() const { return processBlockHelper_; } - // These are intended to be used only for testing purposes + private: + // The next two functions are intended to be used only for testing purposes + friend class TestOneOutput; std::vector const& translateFromStoredIndex() const { return translateFromStoredIndex_; } unsigned int nAddedProcesses() const { return nAddedProcesses_; } - private: void setStoredProcessOffset(unsigned int nInputProcesses, std::vector> const& nEntries, std::vector& storedProcessOffset) const; From 05e0ce137c7537794a2dc1b264c653bbd8e28109 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 24 Jun 2021 23:04:53 +0200 Subject: [PATCH 15/26] Change override to final --- FWCore/Common/interface/ProcessBlockHelper.h | 18 +++++++++--------- .../Common/interface/SubProcessBlockHelper.h | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/FWCore/Common/interface/ProcessBlockHelper.h b/FWCore/Common/interface/ProcessBlockHelper.h index bce4a1c729ed7..c3a97d494f1a8 100644 --- a/FWCore/Common/interface/ProcessBlockHelper.h +++ b/FWCore/Common/interface/ProcessBlockHelper.h @@ -18,15 +18,15 @@ namespace edm { class ProcessBlockHelper : public ProcessBlockHelperBase { public: - ProcessBlockHelperBase const* topProcessBlockHelper() const override; - std::vector const& topProcessesWithProcessBlockProducts() const override; - unsigned int nProcessesInFirstFile() const override; - std::vector> const& processBlockCacheIndices() const override; - std::vector> const& nEntries() const override; - std::vector const& cacheIndexVectorsPerFile() const override; - std::vector const& cacheEntriesPerFile() const override; - unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const override; - unsigned int outerOffset() const override; + ProcessBlockHelperBase const* topProcessBlockHelper() const final; + std::vector const& topProcessesWithProcessBlockProducts() const final; + unsigned int nProcessesInFirstFile() const final; + std::vector> const& processBlockCacheIndices() const final; + std::vector> const& nEntries() const final; + std::vector const& cacheIndexVectorsPerFile() const final; + std::vector const& cacheEntriesPerFile() const final; + unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const final; + unsigned int outerOffset() const final; bool initializedFromInput() const { return initializedFromInput_; } diff --git a/FWCore/Common/interface/SubProcessBlockHelper.h b/FWCore/Common/interface/SubProcessBlockHelper.h index ed17a83aa4069..a99534cf01e25 100644 --- a/FWCore/Common/interface/SubProcessBlockHelper.h +++ b/FWCore/Common/interface/SubProcessBlockHelper.h @@ -16,15 +16,15 @@ namespace edm { class SubProcessBlockHelper : public ProcessBlockHelperBase { public: - ProcessBlockHelperBase const* topProcessBlockHelper() const override; - std::vector const& topProcessesWithProcessBlockProducts() const override; - unsigned int nProcessesInFirstFile() const override; - std::vector> const& processBlockCacheIndices() const override; - std::vector> const& nEntries() const override; - std::vector const& cacheIndexVectorsPerFile() const override; - std::vector const& cacheEntriesPerFile() const override; - unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const override; - unsigned int outerOffset() const override; + ProcessBlockHelperBase const* topProcessBlockHelper() const final; + std::vector const& topProcessesWithProcessBlockProducts() const final; + unsigned int nProcessesInFirstFile() const final; + std::vector> const& processBlockCacheIndices() const final; + std::vector> const& nEntries() const final; + std::vector const& cacheIndexVectorsPerFile() const final; + std::vector const& cacheEntriesPerFile() const final; + unsigned int processBlockIndex(std::string const& processName, EventToProcessBlockIndexes const&) const final; + unsigned int outerOffset() const final; void updateFromParentProcess(ProcessBlockHelperBase const& parentProcessBlockHelper, ProductRegistry const&); From bac65d12ac463806029446c3a364e00998771bb3 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 25 Jun 2021 00:22:49 +0200 Subject: [PATCH 16/26] Use pass by value instead of && --- FWCore/Common/interface/ProcessBlockHelper.h | 6 +++--- FWCore/Common/src/ProcessBlockHelper.cc | 8 ++++---- IOPool/Input/src/RootFile.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/FWCore/Common/interface/ProcessBlockHelper.h b/FWCore/Common/interface/ProcessBlockHelper.h index c3a97d494f1a8..457e61c3511e1 100644 --- a/FWCore/Common/interface/ProcessBlockHelper.h +++ b/FWCore/Common/interface/ProcessBlockHelper.h @@ -44,7 +44,7 @@ namespace edm { void initializeFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper); void fillFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper, - std::vector&& nEntries); + std::vector const& nEntries); void clearAfterOutputFilesClose(); @@ -56,9 +56,9 @@ namespace edm { void fillFromPrimaryInputWhenNotEmpty(std::vector const& storedProcesses, std::vector const& storedCacheIndices, - std::vector&& nEntries); + std::vector const& nEntries); - void fillEntriesFromPrimaryInput(std::vector&& nEntries); + void fillEntriesFromPrimaryInput(std::vector nEntries); // A general comment about this class and its data members. // It was initially written to handle cases where all ProcessBlock diff --git a/FWCore/Common/src/ProcessBlockHelper.cc b/FWCore/Common/src/ProcessBlockHelper.cc index 7f0d65473d685..5e10ff41a1867 100644 --- a/FWCore/Common/src/ProcessBlockHelper.cc +++ b/FWCore/Common/src/ProcessBlockHelper.cc @@ -130,7 +130,7 @@ namespace edm { } void ProcessBlockHelper::fillFromPrimaryInput(StoredProcessBlockHelper const& storedProcessBlockHelper, - std::vector&& nEntries) { + std::vector const& nEntries) { // I've written this so it will continue to work even if we someday relax // the strict merging requirement in the ProductRegistry (there // is a little extra complexity that may be unnecessary...). @@ -145,7 +145,7 @@ namespace edm { return; } else if (!storedProcesses.empty()) { // Subsequent input file with ProcessBlock products - fillFromPrimaryInputWhenNotEmpty(storedProcesses, storedCacheIndices, std::move(nEntries)); + fillFromPrimaryInputWhenNotEmpty(storedProcesses, storedCacheIndices, nEntries); } else if (storedProcesses.empty()) { // Subsequent input file without ProcessBlock products processBlockCacheIndices_.emplace_back(nProcessesInFirstFile_, invalidCacheIndex()); @@ -223,7 +223,7 @@ namespace edm { void ProcessBlockHelper::fillFromPrimaryInputWhenNotEmpty(std::vector const& storedProcesses, std::vector const& storedCacheIndices, - std::vector&& nEntries) { + std::vector const& nEntries) { assert(nProcessesInFirstFile_ <= processesWithProcessBlockProducts().size()); // Calculate a translation from an index into the process names from the first file @@ -268,7 +268,7 @@ namespace edm { fillEntriesFromPrimaryInput(std::move(newNEntries)); } - void ProcessBlockHelper::fillEntriesFromPrimaryInput(std::vector&& nEntries) { + void ProcessBlockHelper::fillEntriesFromPrimaryInput(std::vector nEntries) { unsigned int entriesThisFile = 0; for (auto const& entries : nEntries) { entriesThisFile += entries; diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index 75c0587357a81..76e82cd4d1372 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -1653,7 +1653,7 @@ namespace edm { for (auto const& processBlockTree : processBlockTrees_) { nEntries.push_back(processBlockTree->entries()); } - processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, std::move(nEntries)); + processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, nEntries); storedProcessBlockHelper_ = std::make_unique(); // propagate_const has no reset() function } From ce8b3a5d3dd77288e625cb9c9dd503ab286e4fed Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 25 Jun 2021 17:10:14 +0200 Subject: [PATCH 17/26] Test new set function --- .../Provenance/test/StoredProcessBlockHelper_t.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp index 4ba3f7aaa1406..2a07094892d18 100644 --- a/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp +++ b/DataFormats/Provenance/test/StoredProcessBlockHelper_t.cpp @@ -4,7 +4,6 @@ #include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" #include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" -#include #include #include @@ -14,10 +13,6 @@ TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts().empty()); REQUIRE(storedProcessBlockHelper.processBlockCacheIndices().empty()); - edm::StoredProcessBlockHelper const* storedProcessBlockHelperConstPtr = &storedProcessBlockHelper; - REQUIRE(storedProcessBlockHelperConstPtr->processesWithProcessBlockProducts().empty()); - REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); - edm::EventToProcessBlockIndexes eventToProcessBlockIndexes; REQUIRE(eventToProcessBlockIndexes.index() == 0); eventToProcessBlockIndexes.setIndex(2); @@ -26,18 +21,16 @@ TEST_CASE("StoredProcessBlockHelper", "[StoredProcessBlockHelper]") { SECTION("Constructor") { std::vector testStrings{"test1", "test2", "test3"}; - edm::StoredProcessBlockHelper storedProcessBlockHelper(testStrings); REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts() == testStrings); REQUIRE(storedProcessBlockHelper.processBlockCacheIndices().empty()); - edm::StoredProcessBlockHelper const* storedProcessBlockHelperConstPtr = &storedProcessBlockHelper; - REQUIRE(storedProcessBlockHelperConstPtr->processesWithProcessBlockProducts() == testStrings); - REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices().empty()); + std::vector testStrings2{"test1", "test2", "test3", "test4"}; + storedProcessBlockHelper.setProcessesWithProcessBlockProducts(testStrings2); + REQUIRE(storedProcessBlockHelper.processesWithProcessBlockProducts() == testStrings2); std::vector testIndices{1, 10, 100}; storedProcessBlockHelper.setProcessBlockCacheIndices(testIndices); REQUIRE(storedProcessBlockHelper.processBlockCacheIndices() == testIndices); - REQUIRE(storedProcessBlockHelperConstPtr->processBlockCacheIndices() == testIndices); } } From 390d9670cfe3b49b740f4072b4d3c56d787ff016 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 25 Jun 2021 19:10:00 +0200 Subject: [PATCH 18/26] Use ranged-for loops --- FWCore/Common/src/OutputProcessBlockHelper.cc | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/FWCore/Common/src/OutputProcessBlockHelper.cc b/FWCore/Common/src/OutputProcessBlockHelper.cc index 0e695d51de4ae..0b3c8486ef2c5 100644 --- a/FWCore/Common/src/OutputProcessBlockHelper.cc +++ b/FWCore/Common/src/OutputProcessBlockHelper.cc @@ -188,17 +188,19 @@ namespace edm { void OutputProcessBlockHelper::setStoredProcessOffset(unsigned int nInputProcesses, std::vector> const& nEntries, std::vector& storedProcessOffset) const { - for (unsigned int iStored = 0; iStored < nInputProcesses + 1; ++iStored) { - storedProcessOffset[iStored] = 0; + unsigned int iStored = 0; + for (auto& offset : storedProcessOffset) { + offset = 0; // loop over earlier processes for (unsigned int jStored = 0; jStored < iStored; ++jStored) { unsigned int indexInEventProcessor = translateFromStoredIndex_[jStored]; // loop over input files for (auto const& entries : nEntries) { assert(indexInEventProcessor < entries.size()); - storedProcessOffset[iStored] += entries[indexInEventProcessor]; + offset += entries[indexInEventProcessor]; } } + ++iStored; } } @@ -206,12 +208,14 @@ namespace edm { unsigned int nInputProcesses, std::vector> const& nEntries, std::vector& processOffset) const { - for (unsigned int iStored = 0; iStored < nInputProcesses; ++iStored) { - processOffset[iStored] = 0; + unsigned int iStored = 0; + for (auto& offset : processOffset) { + offset = 0; unsigned int iProcess = translateFromStoredIndex_[iStored]; for (unsigned int jProcess = 0; jProcess < iProcess; ++jProcess) { - processOffset[iStored] += nEntries[iFile][jProcess]; + offset += nEntries[iFile][jProcess]; } + ++iStored; } } @@ -220,14 +224,16 @@ namespace edm { unsigned int nInputProcesses, std::vector> const& nEntries, std::vector& storedFileInProcessOffset) const { - for (unsigned int iStored = 0; iStored < nInputProcesses; ++iStored) { - storedFileInProcessOffset[iStored] = 0; + unsigned int iStored = 0; + for (auto& offset : storedFileInProcessOffset) { + offset = 0; unsigned int indexInEventProcessor = translateFromStoredIndex_[iStored]; // loop over input files before current input file for (unsigned int jFile = 0; jFile < iFile; ++jFile) { assert(indexInEventProcessor < nEntries[jFile].size()); - storedFileInProcessOffset[iStored] += nEntries[jFile][indexInEventProcessor]; + offset += nEntries[jFile][indexInEventProcessor]; } + ++iStored; } } From 02dbc6799e0f7c10f80e9d8905734a82c7a8ce9a Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 25 Jun 2021 22:47:17 +0200 Subject: [PATCH 19/26] Don't use moved object after move --- FWCore/Framework/src/SubProcess.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/FWCore/Framework/src/SubProcess.cc b/FWCore/Framework/src/SubProcess.cc index e38a222a37541..ab3e793ebe01d 100644 --- a/FWCore/Framework/src/SubProcess.cc +++ b/FWCore/Framework/src/SubProcess.cc @@ -510,6 +510,7 @@ namespace edm { ProcessBlockTransitionInfo transitionInfo(processBlockPrincipal); if (parentProducedProductIsKept(parentPrincipal, processBlockPrincipal)) { + auto& taskGroup = *iHolder.group(); auto runEndProcessBlock = make_waiting_task([this, iWait = std::move(iHolder), info = transitionInfo, cleaningUpAfterException]( std::exception_ptr const* iPtr) mutable { @@ -526,7 +527,7 @@ namespace edm { std::move(iWait), *schedule_, info, serviceToken_, subProcesses_, cleaningUpAfterException); } }); - WaitingTaskHolder holder(*iHolder.group(), runEndProcessBlock); + WaitingTaskHolder holder(taskGroup, runEndProcessBlock); auto runWriteProcessBlock = make_waiting_task([this, iWait = std::move(holder)](std::exception_ptr const* iPtr) mutable { @@ -536,7 +537,7 @@ namespace edm { writeProcessBlockAsync(iWait, ProcessBlockType::Input); } }); - WaitingTaskHolder writeHolder(*iHolder.group(), runWriteProcessBlock); + WaitingTaskHolder writeHolder(taskGroup, runWriteProcessBlock); ProcessBlockPrincipal& inputProcessBlockPrincipal = principalCache_.inputProcessBlockPrincipal(); inputProcessBlockPrincipal.fillProcessBlockPrincipal(parentPrincipal.processName(), parentPrincipal.reader()); From ec4699b238040d2dec4c8c123e2473765611db1e Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Mon, 28 Jun 2021 21:14:36 +0200 Subject: [PATCH 20/26] expectedByRun make first element run 1, use at --- .../test/stubs/TestGlobalAnalyzers.cc | 34 ++++++------- .../Framework/test/stubs/TestGlobalFilters.cc | 17 +++---- .../test/stubs/TestGlobalProducers.cc | 17 +++---- .../test/stubs/TestLimitedAnalyzers.cc | 17 +++---- .../test/stubs/TestLimitedFilters.cc | 17 +++---- .../test/stubs/TestLimitedProducers.cc | 17 +++---- .../Framework/test/stubs/TestOneAnalyzers.cc | 17 +++---- FWCore/Framework/test/stubs/TestOneFilters.cc | 17 +++---- .../Framework/test/stubs/TestOneProducers.cc | 17 +++---- .../test/stubs/TestStreamAnalyzers.cc | 16 +++---- .../Framework/test/stubs/TestStreamFilters.cc | 16 +++---- .../test/stubs/TestStreamProducers.cc | 16 +++---- .../test/testLooperEventNavigation2_cfg.py | 2 +- .../test/testLooperEventNavigation3_cfg.py | 2 +- .../test/testProcessBlockDropOnInput_cfg.py | 4 +- .../testProcessBlockMergeOfMergedFiles_cfg.py | 2 +- .../test/testProcessBlockMerge_cfg.py | 2 +- ...estProcessBlockNOMergeOfMergedFiles_cfg.py | 2 +- .../test/testProcessBlockNonStrict2_cfg.py | 4 +- .../test/testProcessBlockNonStrict3_cfg.py | 4 +- .../test/testProcessBlockNonStrict_cfg.py | 4 +- .../test/testProcessBlockRead2_cfg.py | 2 +- .../testProcessBlockReadDropOnOutput2_cfg.py | 2 +- .../testProcessBlockReadDropOnOutput_cfg.py | 2 +- .../testProcessBlockReadThreeFileInput_cfg.py | 4 +- .../test/testProcessBlockRead_cfg.py | 48 +++++++++---------- .../testProcessBlockSubProcessLooper_cfg.py | 6 +-- .../testProcessBlockSubProcessRead1_cfg.py | 6 +-- .../testProcessBlockSubProcessRead2_cfg.py | 8 ++-- .../test/testProcessBlockSubProcess_cfg.py | 6 +-- .../testProcessBlockThreeFileInput_cfg.py | 4 +- 31 files changed, 171 insertions(+), 161 deletions(-) diff --git a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc index 3c68948e3c0f4..7236259ec22c8 100644 --- a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc @@ -528,22 +528,23 @@ namespace edmtest { void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; @@ -658,11 +659,11 @@ namespace edmtest { << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 0"; } } else { - if (expectedByRun0_[event.run()] != *std::get>(cacheTuple)) { + if (expectedByRun0_.at(event.run() - 1) != *std::get>(cacheTuple)) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockAnalyzerThreeTags::analyze zeroth cached value was " << *std::get>(cacheTuple) << " but it was supposed to be " - << expectedByRun0_[event.run()]; + << expectedByRun0_.at(event.run() - 1); } } if (expectedByRun1_.empty()) { @@ -671,10 +672,11 @@ namespace edmtest { << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 1"; } } else { - if (expectedByRun1_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun1_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockAnalyzerThreeTags::analyze first cached value was " - << std::get<1>(cacheTuple)->value_ << " but it was supposed to be " << expectedByRun1_[event.run()]; + << std::get<1>(cacheTuple)->value_ << " but it was supposed to be " + << expectedByRun1_.at(event.run() - 1); } } if (expectedByRun2_.empty()) { @@ -683,12 +685,12 @@ namespace edmtest { << "InputProcessBlockAnalyzerThreeTags::analyze expected invalid CacheHandle for cache 2"; } } else { - if (expectedByRun2_[event.run()] != + if (expectedByRun2_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockAnalyzerThreeTags::analyze second cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun2_[event.run()]; + << " but it was supposed to be " << expectedByRun2_.at(event.run() - 1); } } ++transitions_; @@ -757,12 +759,12 @@ namespace edmtest { void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockAnalyzerReuseCache::analyze cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestGlobalFilters.cc b/FWCore/Framework/test/stubs/TestGlobalFilters.cc index 5845b4ad7c5eb..573198df716d8 100644 --- a/FWCore/Framework/test/stubs/TestGlobalFilters.cc +++ b/FWCore/Framework/test/stubs/TestGlobalFilters.cc @@ -840,22 +840,23 @@ namespace edmtest { bool filter(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestGlobalProducers.cc b/FWCore/Framework/test/stubs/TestGlobalProducers.cc index 0b35ea19f8dd5..3ca4e302c0a4a 100644 --- a/FWCore/Framework/test/stubs/TestGlobalProducers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalProducers.cc @@ -899,22 +899,23 @@ namespace edmtest { void produce(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc index eef2fac9615c1..95f80b3f4ec56 100644 --- a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc @@ -496,22 +496,23 @@ namespace edmtest { void analyze(edm::StreamID, edm::Event const& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestLimitedFilters.cc b/FWCore/Framework/test/stubs/TestLimitedFilters.cc index 4af2499a561fa..4af5ee927c9ac 100644 --- a/FWCore/Framework/test/stubs/TestLimitedFilters.cc +++ b/FWCore/Framework/test/stubs/TestLimitedFilters.cc @@ -876,22 +876,23 @@ namespace edmtest { bool filter(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestLimitedProducers.cc b/FWCore/Framework/test/stubs/TestLimitedProducers.cc index 300a816255c6e..c8f03b61e9ed0 100644 --- a/FWCore/Framework/test/stubs/TestLimitedProducers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedProducers.cc @@ -940,22 +940,23 @@ namespace edmtest { void produce(edm::StreamID, edm::Event& event, edm::EventSetup const&) const override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc index fb5b70c650d47..0d9fa7e6f7fe9 100644 --- a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc @@ -484,22 +484,23 @@ namespace edmtest { void analyze(edm::Event const& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntAnalyzer::analyze cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } diff --git a/FWCore/Framework/test/stubs/TestOneFilters.cc b/FWCore/Framework/test/stubs/TestOneFilters.cc index b684185cc36e6..32831a4a2dfb7 100644 --- a/FWCore/Framework/test/stubs/TestOneFilters.cc +++ b/FWCore/Framework/test/stubs/TestOneFilters.cc @@ -638,22 +638,23 @@ namespace edmtest { bool filter(edm::Event& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntFilter::filter cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestOneProducers.cc b/FWCore/Framework/test/stubs/TestOneProducers.cc index a79142f254bb7..00cecde3d84b0 100644 --- a/FWCore/Framework/test/stubs/TestOneProducers.cc +++ b/FWCore/Framework/test/stubs/TestOneProducers.cc @@ -662,22 +662,23 @@ namespace edmtest { void produce(edm::Event& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != *std::get>(cacheTuple)) { - throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " - << *std::get>(cacheTuple) - << " but it was supposed to be " << expectedByRun_[event.run()]; + if (expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { + throw cms::Exception("UnexpectedValue") + << "InputProcessBlockIntProducer::produce cached value was " + << *std::get>(cacheTuple) << " but it was supposed to be " + << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } ++transitions_; diff --git a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc index acf31360c1b19..f3aabf78f28c8 100644 --- a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc @@ -601,12 +601,12 @@ namespace edmtest { void analyze(edm::Event const& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzer::analyze cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } // Force events to be processed concurrently @@ -739,23 +739,23 @@ namespace edmtest { auto cacheTuple = processBlockCaches(event); auto testGlobalCache = globalCache(); if (!testGlobalCache->expectedByRun_.empty()) { - if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzerG::analyze cached value was " << *std::get>(cacheTuple) << " but it was supposed to be " - << testGlobalCache->expectedByRun_[event.run()]; + << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzerG::analyze second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntAnalyzerG::analyze third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } } ++testGlobalCache->transitions_; diff --git a/FWCore/Framework/test/stubs/TestStreamFilters.cc b/FWCore/Framework/test/stubs/TestStreamFilters.cc index d8a33f1b84a28..fc09f6cd704e8 100644 --- a/FWCore/Framework/test/stubs/TestStreamFilters.cc +++ b/FWCore/Framework/test/stubs/TestStreamFilters.cc @@ -939,12 +939,12 @@ namespace edmtest { bool filter(edm::Event& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilter::filter cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } // Force events to be processed concurrently @@ -1078,23 +1078,23 @@ namespace edmtest { auto cacheTuple = processBlockCaches(event); auto testGlobalCache = globalCache(); if (!testGlobalCache->expectedByRun_.empty()) { - if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilterG::filter cached value was " << *std::get>(cacheTuple) << " but it was supposed to be " - << testGlobalCache->expectedByRun_[event.run()]; + << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilterG::filter second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntFilterG::filter third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } } ++testGlobalCache->transitions_; diff --git a/FWCore/Framework/test/stubs/TestStreamProducers.cc b/FWCore/Framework/test/stubs/TestStreamProducers.cc index c5197f11893f1..9a8c0d59c5d5c 100644 --- a/FWCore/Framework/test/stubs/TestStreamProducers.cc +++ b/FWCore/Framework/test/stubs/TestStreamProducers.cc @@ -1014,12 +1014,12 @@ namespace edmtest { void produce(edm::Event& event, edm::EventSetup const&) override { auto cacheTuple = processBlockCaches(event); if (!expectedByRun_.empty()) { - if (expectedByRun_[event.run()] != + if (expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducer::produce cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << expectedByRun_[event.run()]; + << " but it was supposed to be " << expectedByRun_.at(event.run() - 1); } } // Force events to be processed concurrently @@ -1152,23 +1152,23 @@ namespace edmtest { auto cacheTuple = processBlockCaches(event); auto testGlobalCache = globalCache(); if (!testGlobalCache->expectedByRun_.empty()) { - if (testGlobalCache->expectedByRun_[event.run()] != *std::get>(cacheTuple)) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != *std::get>(cacheTuple)) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducerG::produce cached value was " << *std::get>(cacheTuple) << " but it was supposed to be " - << testGlobalCache->expectedByRun_[event.run()]; + << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != std::get<1>(cacheTuple)->value_) { + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get<1>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducerG::produce second cached value was " << std::get<1>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } - if (testGlobalCache->expectedByRun_[event.run()] != + if (testGlobalCache->expectedByRun_.at(event.run() - 1) != std::get>(cacheTuple)->value_) { throw cms::Exception("UnexpectedValue") << "InputProcessBlockIntProducerG::produce third cached value was " << std::get>(cacheTuple)->value_ - << " but it was supposed to be " << testGlobalCache->expectedByRun_[event.run()]; + << " but it was supposed to be " << testGlobalCache->expectedByRun_.at(event.run() - 1); } } ++testGlobalCache->transitions_; diff --git a/FWCore/Integration/test/testLooperEventNavigation2_cfg.py b/FWCore/Integration/test/testLooperEventNavigation2_cfg.py index d5cb16155cbda..1c03c90078f3f 100644 --- a/FWCore/Integration/test/testLooperEventNavigation2_cfg.py +++ b/FWCore/Integration/test/testLooperEventNavigation2_cfg.py @@ -17,7 +17,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(16519) ) diff --git a/FWCore/Integration/test/testLooperEventNavigation3_cfg.py b/FWCore/Integration/test/testLooperEventNavigation3_cfg.py index 089b6ebbca527..2d9f673d05ea4 100644 --- a/FWCore/Integration/test/testLooperEventNavigation3_cfg.py +++ b/FWCore/Integration/test/testLooperEventNavigation3_cfg.py @@ -18,7 +18,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221) ) diff --git a/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py b/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py index dd872ebbc3ae3..e408dc595c3d5 100644 --- a/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py +++ b/FWCore/Integration/test/testProcessBlockDropOnInput_cfg.py @@ -29,7 +29,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(15440), consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), @@ -43,7 +43,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(0), consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), diff --git a/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py index 15a6f0f754c18..b53c7c248b507 100644 --- a/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py +++ b/FWCore/Integration/test/testProcessBlockMergeOfMergedFiles_cfg.py @@ -21,7 +21,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221) ) diff --git a/FWCore/Integration/test/testProcessBlockMerge_cfg.py b/FWCore/Integration/test/testProcessBlockMerge_cfg.py index dff8a69d3d0a7..6ba9dbee0c80b 100644 --- a/FWCore/Integration/test/testProcessBlockMerge_cfg.py +++ b/FWCore/Integration/test/testProcessBlockMerge_cfg.py @@ -22,7 +22,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7700), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7700), expectedSum = cms.int32(33) ) diff --git a/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py b/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py index 9c6bc83d7fb87..210b4dd8a4cea 100644 --- a/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py +++ b/FWCore/Integration/test/testProcessBlockNOMergeOfMergedFiles_cfg.py @@ -22,7 +22,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221) ) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py index eb7dec9550ae8..ded685c89745d 100644 --- a/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py +++ b/FWCore/Integration/test/testProcessBlockNonStrict2_cfg.py @@ -36,7 +36,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(47384) ) @@ -48,7 +48,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(1064) ) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py index 93b8047ffed5b..75f07da947fb2 100644 --- a/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py +++ b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py @@ -39,7 +39,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - #expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + #expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(532) ) @@ -51,7 +51,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - #expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + #expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(532) ) diff --git a/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py index 72b46ceef0f87..ae7fe3f40d2ce 100644 --- a/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py +++ b/FWCore/Integration/test/testProcessBlockNonStrict_cfg.py @@ -32,7 +32,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(31412) ) @@ -44,7 +44,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(532) ) diff --git a/FWCore/Integration/test/testProcessBlockRead2_cfg.py b/FWCore/Integration/test/testProcessBlockRead2_cfg.py index 6b491d77f4062..1981ea777c553 100644 --- a/FWCore/Integration/test/testProcessBlockRead2_cfg.py +++ b/FWCore/Integration/test/testProcessBlockRead2_cfg.py @@ -21,7 +21,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221) ) diff --git a/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py b/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py index f839d56780718..e13d29f196cf8 100644 --- a/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py +++ b/FWCore/Integration/test/testProcessBlockReadDropOnOutput2_cfg.py @@ -16,7 +16,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(15440) ) diff --git a/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py b/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py index 17556de910b06..3555568296cfd 100644 --- a/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py +++ b/FWCore/Integration/test/testProcessBlockReadDropOnOutput_cfg.py @@ -16,7 +16,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(15440) ) diff --git a/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py b/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py index c59d65fca545f..9d8be6e893aae 100644 --- a/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py +++ b/FWCore/Integration/test/testProcessBlockReadThreeFileInput_cfg.py @@ -15,7 +15,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(24193) ) @@ -25,7 +25,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(1020) ) diff --git a/FWCore/Integration/test/testProcessBlockRead_cfg.py b/FWCore/Integration/test/testProcessBlockRead_cfg.py index 12da104b98780..f0891560242b8 100644 --- a/FWCore/Integration/test/testProcessBlockRead_cfg.py +++ b/FWCore/Integration/test/testProcessBlockRead_cfg.py @@ -44,7 +44,7 @@ process.readProcessBlocks = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzer", consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), sleepTime = cms.uint32(10000) ) @@ -54,7 +54,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), sleepTime = cms.uint32(10000) ) @@ -65,7 +65,7 @@ process.readProcessBlocksStreamFilter = cms.EDFilter("edmtest::stream::InputProcessBlockIntFilter", consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), sleepTime = cms.uint32(10000) ) @@ -75,7 +75,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), sleepTime = cms.uint32(10000) ) @@ -83,7 +83,7 @@ process.readProcessBlocksStreamProducer = cms.EDProducer("edmtest::stream::InputProcessBlockIntProducer", consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), sleepTime = cms.uint32(10000) ) @@ -93,7 +93,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), sleepTime = cms.uint32(10000) ) @@ -104,7 +104,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlock") @@ -116,7 +116,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77) ) @@ -126,7 +126,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77) ) @@ -136,7 +136,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77) ) @@ -146,7 +146,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77) ) @@ -156,7 +156,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77) ) @@ -166,7 +166,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), concurrencyLimit = cms.untracked.uint32(4) ) @@ -177,7 +177,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), concurrencyLimit = cms.untracked.uint32(4) ) @@ -188,7 +188,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22), + expectedByRun = cms.vint32(11, 22), expectedSum = cms.int32(77), concurrencyLimit = cms.untracked.uint32(4) ) @@ -205,9 +205,9 @@ consumesEndProcessBlock1 = cms.InputTag("doesNotExist", ""), consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun0 = cms.vint32(0, 11, 22), + expectedByRun0 = cms.vint32(11, 22), expectedByRun1 = cms.vint32(), - expectedByRun2 = cms.vint32(0, 44, 44) + expectedByRun2 = cms.vint32(44, 44) ) process.readProcessBlocksExplicitProcess = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", @@ -218,9 +218,9 @@ consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlockB", "", "PROD1"), consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockB", "", "MERGE"), consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockB", "", "MERGE"), - expectedByRun0 = cms.vint32(0, 88, 88), - expectedByRun1 = cms.vint32(0, 55, 77), - expectedByRun2 = cms.vint32(0, 88, 88) + expectedByRun0 = cms.vint32(88, 88), + expectedByRun1 = cms.vint32(55, 77), + expectedByRun2 = cms.vint32(88, 88) ) process.readProcessBlocksExplicitProcess2 = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", @@ -231,16 +231,16 @@ consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlock", "", "PROD1"), consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", "", "MERGE"), consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", "", "MERGE"), - expectedByRun0 = cms.vint32(0, 11, 22), - expectedByRun1 = cms.vint32(0, 11, 22), - expectedByRun2 = cms.vint32(0, 44, 44) + expectedByRun0 = cms.vint32(11, 22), + expectedByRun1 = cms.vint32(11, 22), + expectedByRun2 = cms.vint32(44, 44) ) process.readProcessBlocksReuseCache = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerReuseCache", transitions = cms.int32(8), consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), - expectedByRun = cms.vint32(0, 11, 11) + expectedByRun = cms.vint32(11, 11) ) process.p = cms.Path(process.intProducerBeginProcessBlockR * diff --git a/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py index c9d06094586b3..bf2a69404cf66 100644 --- a/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py +++ b/FWCore/Integration/test/testProcessBlockSubProcessLooper_cfg.py @@ -128,7 +128,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(24663) ) @@ -138,7 +138,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 44, 44, 444, 444), + expectedByRun = cms.vint32(44, 44, 444, 444), expectedSum = cms.int32(1464) ) @@ -148,7 +148,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644), + expectedByRun = cms.vint32(644, 644, 644, 644), expectedSum = cms.int32(1464) ) diff --git a/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py index 6ba4501dfb72d..23c971a85e5c9 100644 --- a/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py +++ b/FWCore/Integration/test/testProcessBlockSubProcessRead1_cfg.py @@ -21,7 +21,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221), expectedFillerSum = cms.untracked.int32(23199), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), @@ -34,7 +34,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 44000, 44000, 44000, 44000), + expectedByRun = cms.vint32(44000, 44000, 44000, 44000), expectedSum = cms.int32(488), expectedFillerSum = cms.untracked.int32(132000), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), @@ -47,7 +47,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 55000, 55000, 55000, 55000), + expectedByRun = cms.vint32(55000, 55000, 55000, 55000), expectedSum = cms.int32(488), expectedFillerSum = cms.untracked.int32(165000), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockR"), diff --git a/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py index bc4ee6ad11723..51296e8bf0ae5 100644 --- a/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py +++ b/FWCore/Integration/test/testProcessBlockSubProcessRead2_cfg.py @@ -21,7 +21,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221), expectedFillerSum = cms.untracked.int32(23199), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), @@ -34,7 +34,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockT", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 44000, 44000, 44000, 44000), + expectedByRun = cms.vint32(44000, 44000, 44000, 44000), expectedSum = cms.int32(488), expectedFillerSum = cms.untracked.int32(132000), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockT"), @@ -47,7 +47,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockR", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 55000, 55000, 55000, 55000), + expectedByRun = cms.vint32(55000, 55000, 55000, 55000), expectedSum = cms.int32(488), expectedFillerSum = cms.untracked.int32(165000), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockR"), @@ -60,7 +60,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockRA", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 1100000, 1100000, 1100000, 1100000), + expectedByRun = cms.vint32(1100000, 1100000, 1100000, 1100000), expectedSum = cms.int32(488), expectedFillerSum = cms.untracked.int32(3300000), consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlockRA"), diff --git a/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py b/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py index 29bc47b1b7046..88505194cdb03 100644 --- a/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py +++ b/FWCore/Integration/test/testProcessBlockSubProcess_cfg.py @@ -118,7 +118,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400), + expectedByRun = cms.vint32(11, 22, 3300, 4400), expectedSum = cms.int32(8221) ) @@ -130,7 +130,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 44, 44, 444, 444), + expectedByRun = cms.vint32(44, 44, 444, 444), expectedSum = cms.int32(488) ) @@ -142,7 +142,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644), + expectedByRun = cms.vint32(644, 644, 644, 644), expectedSum = cms.int32(488) ) diff --git a/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py b/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py index 5a82284617597..0032718691728 100644 --- a/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py +++ b/FWCore/Integration/test/testProcessBlockThreeFileInput_cfg.py @@ -24,7 +24,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 11, 22, 3300, 4400, 7707), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), expectedSum = cms.int32(24193) ) @@ -34,7 +34,7 @@ consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), - expectedByRun = cms.vint32(0, 644, 644, 644, 644, 844), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), expectedSum = cms.int32(1020) ) From d08a9166b621f948452ec97cd8731a7b8f793ba5 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Mon, 28 Jun 2021 21:39:30 +0200 Subject: [PATCH 21/26] Remove CMS_THREAD_SAFE from atomic variables --- FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc | 8 ++++---- FWCore/Framework/test/stubs/TestGlobalFilters.cc | 2 +- FWCore/Framework/test/stubs/TestGlobalProducers.cc | 2 +- FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc | 2 +- FWCore/Framework/test/stubs/TestLimitedFilters.cc | 2 +- FWCore/Framework/test/stubs/TestLimitedProducers.cc | 2 +- FWCore/Framework/test/stubs/TestOneAnalyzers.cc | 2 +- FWCore/Framework/test/stubs/TestOneFilters.cc | 2 +- FWCore/Framework/test/stubs/TestOneProducers.cc | 2 +- FWCore/Framework/test/stubs/TestStreamAnalyzers.cc | 4 ++-- FWCore/Framework/test/stubs/TestStreamFilters.cc | 4 ++-- FWCore/Framework/test/stubs/TestStreamProducers.cc | 4 ++-- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc index 7236259ec22c8..c0ebb4df48f47 100644 --- a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc @@ -570,7 +570,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; @@ -714,7 +714,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd1_; edm::EDGetTokenT getTokenBegin2_; edm::EDGetTokenT getTokenEnd2_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun0_; std::vector expectedByRun1_; @@ -784,7 +784,7 @@ namespace edmtest { private: edm::EDGetTokenT getTokenBegin_; edm::EDGetTokenT getTokenEnd_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; }; @@ -815,7 +815,7 @@ namespace edmtest { } private: - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; unsigned int expectedTransitions_{0}; }; diff --git a/FWCore/Framework/test/stubs/TestGlobalFilters.cc b/FWCore/Framework/test/stubs/TestGlobalFilters.cc index 573198df716d8..1465a9060ad7f 100644 --- a/FWCore/Framework/test/stubs/TestGlobalFilters.cc +++ b/FWCore/Framework/test/stubs/TestGlobalFilters.cc @@ -883,7 +883,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestGlobalProducers.cc b/FWCore/Framework/test/stubs/TestGlobalProducers.cc index 3ca4e302c0a4a..ee05190d9d74e 100644 --- a/FWCore/Framework/test/stubs/TestGlobalProducers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalProducers.cc @@ -941,7 +941,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc index 95f80b3f4ec56..55ac918ad1d92 100644 --- a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc @@ -538,7 +538,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestLimitedFilters.cc b/FWCore/Framework/test/stubs/TestLimitedFilters.cc index 4af5ee927c9ac..b5aad0c82510e 100644 --- a/FWCore/Framework/test/stubs/TestLimitedFilters.cc +++ b/FWCore/Framework/test/stubs/TestLimitedFilters.cc @@ -919,7 +919,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestLimitedProducers.cc b/FWCore/Framework/test/stubs/TestLimitedProducers.cc index c8f03b61e9ed0..52a9d734537a3 100644 --- a/FWCore/Framework/test/stubs/TestLimitedProducers.cc +++ b/FWCore/Framework/test/stubs/TestLimitedProducers.cc @@ -982,7 +982,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc index 0d9fa7e6f7fe9..89642cfaa1895 100644 --- a/FWCore/Framework/test/stubs/TestOneAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestOneAnalyzers.cc @@ -562,7 +562,7 @@ namespace edmtest { edm::EDGetTokenT getTokenNotFound2_; edm::EDGetTokenT getTokenNotFound3_; edm::EDGetTokenT getTokenNotFound4_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestOneFilters.cc b/FWCore/Framework/test/stubs/TestOneFilters.cc index 32831a4a2dfb7..0424d494c7b7c 100644 --- a/FWCore/Framework/test/stubs/TestOneFilters.cc +++ b/FWCore/Framework/test/stubs/TestOneFilters.cc @@ -681,7 +681,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestOneProducers.cc b/FWCore/Framework/test/stubs/TestOneProducers.cc index 00cecde3d84b0..25e2a4b0769f3 100644 --- a/FWCore/Framework/test/stubs/TestOneProducers.cc +++ b/FWCore/Framework/test/stubs/TestOneProducers.cc @@ -704,7 +704,7 @@ namespace edmtest { edm::EDGetTokenT getTokenEnd_; edm::EDGetTokenT getTokenBeginM_; edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc index f3aabf78f28c8..5137ede36ee54 100644 --- a/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc +++ b/FWCore/Framework/test/stubs/TestStreamAnalyzers.cc @@ -48,7 +48,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; unsigned int trans_{0}; - CMS_THREAD_SAFE mutable std::atomic m_count{0}; + mutable std::atomic m_count{0}; }; } // namespace cache @@ -640,7 +640,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestStreamFilters.cc b/FWCore/Framework/test/stubs/TestStreamFilters.cc index fc09f6cd704e8..57ac06456e930 100644 --- a/FWCore/Framework/test/stubs/TestStreamFilters.cc +++ b/FWCore/Framework/test/stubs/TestStreamFilters.cc @@ -48,7 +48,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; unsigned int trans_{0}; - CMS_THREAD_SAFE mutable std::atomic m_count{0}; + mutable std::atomic m_count{0}; }; } // namespace cache @@ -979,7 +979,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; diff --git a/FWCore/Framework/test/stubs/TestStreamProducers.cc b/FWCore/Framework/test/stubs/TestStreamProducers.cc index 9a8c0d59c5d5c..e9a64713d1978 100644 --- a/FWCore/Framework/test/stubs/TestStreamProducers.cc +++ b/FWCore/Framework/test/stubs/TestStreamProducers.cc @@ -55,7 +55,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBegin_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; unsigned int trans_{0}; - CMS_THREAD_SAFE mutable std::atomic m_count{0}; + mutable std::atomic m_count{0}; }; } // namespace cache @@ -1053,7 +1053,7 @@ namespace edmtest { CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEnd_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenBeginM_; CMS_THREAD_SAFE mutable edm::EDGetTokenT getTokenEndM_; - CMS_THREAD_SAFE mutable std::atomic transitions_{0}; + mutable std::atomic transitions_{0}; int sum_{0}; unsigned int expectedTransitions_{0}; std::vector expectedByRun_; From 22a00eff970ecec90c31352b434e7d9dab721bde Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Mon, 28 Jun 2021 23:08:02 +0200 Subject: [PATCH 22/26] Use catch2 STATIC_REQUIRE instead of REQUIRE --- .../test_catch2notTP_InputProcessBlock.cc | 58 ++++---- ...atch2notTP_InputProcessBlockCacheHolder.cc | 124 +++++++++--------- 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc b/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc index 13894c38add9d..c2065640125bf 100644 --- a/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc +++ b/FWCore/Framework/test/test_catch2notTP_InputProcessBlock.cc @@ -31,38 +31,38 @@ namespace edmtest { TEST_CASE("test InputProcessBlock", "[InputProcessBlock]") { SECTION("test HasAbility") { - REQUIRE(edmtest::TestAnalyzerWithInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); - REQUIRE(!edmtest::TestAnalyzerWithoutInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); + STATIC_REQUIRE(edmtest::TestAnalyzerWithInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); + STATIC_REQUIRE(!edmtest::TestAnalyzerWithoutInputProcessBlockAbility::HasAbility::kInputProcessBlockCache); } SECTION("test type aliases") { - REQUIRE(std::is_same>()); - REQUIRE(std::is_same()); - REQUIRE(std::is_same::type, - std::unique_ptr>>()); - REQUIRE(std::is_same::type, - edm::stream::impl::dummy_ptr>()); + STATIC_REQUIRE(std::is_same>()); + STATIC_REQUIRE(std::is_same()); + STATIC_REQUIRE(std::is_same::type, + std::unique_ptr>>()); + STATIC_REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); - REQUIRE(std::is_same>()); - REQUIRE(std::is_same()); - REQUIRE(std::is_same::type, - std::unique_ptr>>()); - REQUIRE(std::is_same::type, - edm::stream::impl::dummy_ptr>()); + STATIC_REQUIRE(std::is_same>()); + STATIC_REQUIRE(std::is_same()); + STATIC_REQUIRE(std::is_same::type, + std::unique_ptr>>()); + STATIC_REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); - REQUIRE(std::is_same>()); - REQUIRE(std::is_same()); - REQUIRE(std::is_same::type, - std::unique_ptr>>()); - REQUIRE(std::is_same::type, - edm::stream::impl::dummy_ptr>()); + STATIC_REQUIRE(std::is_same>()); + STATIC_REQUIRE(std::is_same()); + STATIC_REQUIRE(std::is_same::type, + std::unique_ptr>>()); + STATIC_REQUIRE(std::is_same::type, + edm::stream::impl::dummy_ptr>()); } } diff --git a/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc b/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc index 5d2b225bf2109..2b14a4f724e5e 100644 --- a/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc +++ b/FWCore/Framework/test/test_catch2notTP_InputProcessBlockCacheHolder.cc @@ -84,76 +84,76 @@ namespace edmtest { TEST_CASE("test InputProcessBlockCacheHolder", "[InputProcessBlockCacheHolder]") { SECTION("test countTypeInParameterPack") { - REQUIRE(edm::impl::countTypeInParameterPack() == 0); - REQUIRE(edm::impl::countTypeInParameterPack() == - 1); - REQUIRE(edm::impl::countTypeInParameterPack() == - 0); - REQUIRE(edm::impl::countTypeInParameterPack() == 1); - REQUIRE(edm::impl::countTypeInParameterPack() == 1); - REQUIRE(edm::impl::countTypeInParameterPack() == 2); - REQUIRE(edm::impl::countTypeInParameterPack() == 0); - - REQUIRE(edm::impl::countTypeInParameterPack() == 2); - REQUIRE(edm::impl::countTypeInParameterPack() == 1); - REQUIRE(edm::impl::countTypeInParameterPack() == 3); - REQUIRE(edm::impl::countTypeInParameterPack() == 2); - REQUIRE(edm::impl::countTypeInParameterPack() == 1); - REQUIRE(edm::impl::countTypeInParameterPack() == 0); - REQUIRE(edm::impl::countTypeInParameterPack() == 2); - REQUIRE(edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 0); + STATIC_REQUIRE( + edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE( + edm::impl::countTypeInParameterPack() == 0); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 2); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 0); + + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 2); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 3); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 2); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 1); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 0); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 2); + STATIC_REQUIRE(edm::impl::countTypeInParameterPack() == 1); // The following should not compile and I manually verified it doesn't // REQUIRE(edm::impl::countTypeInParameterPack<>() == 0); } SECTION("test indexInputProcessBlockCache") { - REQUIRE( + STATIC_REQUIRE( edm::impl::indexInputProcessBlockCache() == 0); - REQUIRE(edm::impl::indexInputProcessBlockCache() == 0); - REQUIRE(edm::impl::indexInputProcessBlockCache() == 1); - REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); + STATIC_REQUIRE(edm::impl::indexInputProcessBlockCache() == 0); + STATIC_REQUIRE(edm::impl::indexInputProcessBlockCache() == 1); + STATIC_REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); // The following fails compilation if uncommented, tested manually // REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); // REQUIRE(edm::impl::indexInputProcessBlockCache() == 3); From 258dbf26b2f96bcaa31afb2e1dd1bd9a7eca5019 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 1 Jul 2021 00:05:15 +0200 Subject: [PATCH 23/26] Use constexpr variable instead of hard coding --- FWCore/Integration/test/TestOneOutput.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/FWCore/Integration/test/TestOneOutput.cc b/FWCore/Integration/test/TestOneOutput.cc index 671d330e292a2..a8353cd67edb4 100644 --- a/FWCore/Integration/test/TestOneOutput.cc +++ b/FWCore/Integration/test/TestOneOutput.cc @@ -17,6 +17,8 @@ #include #include +constexpr unsigned int kDoNotTest = 0xffffffff; + namespace edm { class TestOneOutput : public one::OutputModule, LuminosityBlockCache> { @@ -281,7 +283,7 @@ namespace edm { << "TestOneOutput::writeProcessBlock unexpected value for translateFromStoredIndex"; } } - if (expectedNAddedProcesses_ != 0xffffffff) { + if (expectedNAddedProcesses_ != kDoNotTest) { if (expectedNAddedProcesses_ != outputProcessBlockHelper().nAddedProcesses()) { throw cms::Exception("TestFailure") << "TestOneOutput::writeProcessBlock unexpected value for nAddedProcesses"; } @@ -401,7 +403,7 @@ namespace edm { desc.addUntracked>("expectedCacheEntriesPerFile2", std::vector()); desc.addUntracked>("expectedOuterOffset", std::vector()); desc.addUntracked>("expectedTranslateFromStoredIndex", std::vector()); - desc.addUntracked("expectedNAddedProcesses", 0xffffffff); + desc.addUntracked("expectedNAddedProcesses", kDoNotTest); desc.addUntracked("expectedProductsFromInputKept", true); descriptions.addDefault(desc); From d12a76b3b013c3cdf421c4dfcdb5d271bc248e92 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Thu, 1 Jul 2021 00:06:43 +0200 Subject: [PATCH 24/26] Fix failure return value in unit test --- FWCore/Integration/test/run_TestProcessBlock.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/FWCore/Integration/test/run_TestProcessBlock.sh b/FWCore/Integration/test/run_TestProcessBlock.sh index 9bca5dcdff707..6254d24fb8cd0 100755 --- a/FWCore/Integration/test/run_TestProcessBlock.sh +++ b/FWCore/Integration/test/run_TestProcessBlock.sh @@ -14,7 +14,7 @@ pushd ${LOCAL_TMP_DIR} grep "Branch.* ProcessBlockHelper " testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? grep "TTree.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? edmFileUtil -t Events -P file:testProcessBlock1.root > testProcessBlock1ContentsE.txt - grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" $? + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 echo "testProcessBlock2" cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock2_cfg.py" $? @@ -51,15 +51,15 @@ pushd ${LOCAL_TMP_DIR} # The ProcessBlock Branches and TTrees should not exist in this case because # all the ProcessBlock products are dropped. Test that here: edmFileUtil -l -t MetaData -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsM.txt - grep "Branch.* ProcessBlockHelper " testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlockHelper branch" $? - grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" $? + grep "Branch.* ProcessBlockHelper " testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlockHelper branch" 1 + grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" 1 edmFileUtil -t Events -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsE.txt - grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" $? + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 # This one intentionally fails because the product content of the # files does not match (strict merging requirements for ProcessBlocks) echo "testProcessBlockFailMerge" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py > /dev/null 2>&1 && die "cmsRun testProcessBlockFailMerge_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py > /dev/null 2>&1 && die "cmsRun testProcessBlockFailMerge_cfg.py" 1 echo "testProcessBlockMerge2" cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge2_cfg.py" $? From bf90eaefdd989f8da54577fdcdcf160272348866 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Wed, 7 Jul 2021 17:52:46 +0200 Subject: [PATCH 25/26] Initialize treePointers_ using the enums --- IOPool/Input/src/RootFile.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index 76e82cd4d1372..8d65a460884a4 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -252,9 +252,10 @@ namespace edm { inputType_(inputType) { hasNewlyDroppedBranch_.fill(false); - treePointers_.emplace_back(&eventTree_); - treePointers_.emplace_back(&lumiTree_); - treePointers_.emplace_back(&runTree_); + treePointers_.resize(3); + treePointers_[InEvent] = &eventTree_; + treePointers_[InLumi] = &lumiTree_; + treePointers_[InRun] = &runTree_; // Read the metadata tree. // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. From 6ecbb2770f75104fb50632bceee21124bc9a7531 Mon Sep 17 00:00:00 2001 From: "W. David Dagenhart" Date: Fri, 9 Jul 2021 00:55:14 +0200 Subject: [PATCH 26/26] Improvements to ProcessBlock test script/BuildFile --- FWCore/Integration/test/BuildFile.xml | 92 ++++++- .../Integration/test/run_TestProcessBlock.sh | 240 ++++++++++++++---- .../test/testProcessBlockNonStrict3_cfg.py | 2 +- 3 files changed, 278 insertions(+), 56 deletions(-) diff --git a/FWCore/Integration/test/BuildFile.xml b/FWCore/Integration/test/BuildFile.xml index 9b420db88563b..b1082508dbdda 100644 --- a/FWCore/Integration/test/BuildFile.xml +++ b/FWCore/Integration/test/BuildFile.xml @@ -77,11 +77,6 @@ - - - - - @@ -434,7 +429,7 @@ - + @@ -450,4 +445,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/FWCore/Integration/test/run_TestProcessBlock.sh b/FWCore/Integration/test/run_TestProcessBlock.sh index 6254d24fb8cd0..18fae01ba0fd3 100755 --- a/FWCore/Integration/test/run_TestProcessBlock.sh +++ b/FWCore/Integration/test/run_TestProcessBlock.sh @@ -1,11 +1,30 @@ #!/bin/bash +set -x +LOCAL_TEST_DIR=${CMSSW_BASE}/src/FWCore/Integration/test +LOCAL_TMP_DIR=${CMSSW_BASE}/tmp/${SCRAM_ARCH} function die { echo Failure $1: status $2 ; exit $2 ; } pushd ${LOCAL_TMP_DIR} +# The tests executed by this bash script are all related and +# it seemed clearest to include them all in the same file. +# These tests are divided into distinct groups. Each time +# the script runs, it will execute only one group of tests. +# The script requires that its first command line argument +# specifies the group to be run. The "if" conditional statements +# below implement this. The BuildFile directs scram to run +# this script once for each group when unit tests are run. +# The BuildFile also specifies the dependencies between the +# groups. In some cases, one group cannot run until another +# group of tests has finished. The purpose of this is to +# allow maximum concurrency while running the tests so the +# tests can run faster. + +if [ $1 -eq 1 ] +then echo "testProcessBlock1" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock1_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock1_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock1_cfg.py &> testProcessBlock1.log || die "cmsRun testProcessBlock1_cfg.py" $? # The MetaData ProcessBlock branch and the TTree should exist to hold the ProcessBlock # data. The Events branch should not exist because there were not any ProcessBlock branches @@ -15,18 +34,30 @@ pushd ${LOCAL_TMP_DIR} grep "TTree.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? edmFileUtil -t Events -P file:testProcessBlock1.root > testProcessBlock1ContentsE.txt grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 +fi +if [ $1 -eq 2 ] +then echo "testProcessBlock2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2_cfg.py &> testProcessBlock2.log || die "cmsRun testProcessBlock2_cfg.py" $? +fi +if [ $1 -eq 3 ] +then echo "testProcessBlock3" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock3_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock3_cfg.py &> testProcessBlock3.log || die "cmsRun testProcessBlock3_cfg.py" $? +fi +if [ $1 -eq 4 ] +then echo "testProcessBlock4" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock4_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock4_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock4_cfg.py &> testProcessBlock4.log || die "cmsRun testProcessBlock4_cfg.py" $? +fi +if [ $1 -eq 5 ] +then echo "testProcessBlockMerge" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge_cfg.py &> testProcessBlockMerge.log || die "cmsRun testProcessBlockMerge_cfg.py" $? # The ProcessBlock Branches and TTrees should exist in this case. Test that here: edmFileUtil -l -t MetaData -P file:testProcessBlockMerge.root > testProcessBlockMContentsM.txt @@ -35,18 +66,24 @@ pushd ${LOCAL_TMP_DIR} grep "TTree.*ProcessBlocksMERGE" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksMERGE TTree" $? edmFileUtil -t Events -P file:testProcessBlockMerge.root > testProcessBlockMContentsE.txt grep "Branch.* EventToProcessBlockIndexes " testProcessBlockMContentsE.txt || die "Check for existence of eventToProcessBlockIndexes branch" $? +fi +if [ $1 -eq 6 ] +then echo "testProcessBlockTEST" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockTEST_cfg.py || die "cmsRun testProcessBlockTEST_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockTEST_cfg.py &> testProcessBlockTEST.log || die "cmsRun testProcessBlockTEST_cfg.py" $? echo "testProcessBlockRead" cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockRead_cfg.py &> testProcessBlockRead.log || die "cmsRun testProcessBlockRead_cfg.py" $? grep "InputProcessBlockIntAnalyzer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntAnalyzer::accessInputProcessBlock was called" $? grep "InputProcessBlockIntFilter::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntFilter::accessInputProcessBlock was called" $? grep "InputProcessBlockIntProducer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntProducer::accessInputProcessBlock was called" $? +fi +if [ $1 -eq 7 ] +then echo "testProcessBlock2Dropped" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2Dropped_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock2Dropped_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock2Dropped_cfg.py &> testProcessBlock2Dropped.log || die "cmsRun testProcessBlock2Dropped_cfg.py" $? # The ProcessBlock Branches and TTrees should not exist in this case because # all the ProcessBlock products are dropped. Test that here: @@ -55,95 +92,183 @@ pushd ${LOCAL_TMP_DIR} grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" 1 edmFileUtil -t Events -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsE.txt grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 +fi +if [ $1 -eq 8 ] +then # This one intentionally fails because the product content of the # files does not match (strict merging requirements for ProcessBlocks) echo "testProcessBlockFailMerge" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py > /dev/null 2>&1 && die "cmsRun testProcessBlockFailMerge_cfg.py" 1 + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py &> testProcessBlockFailMerge.log && die "cmsRun testProcessBlockFailMerge_cfg.py" 1 +fi +if [ $1 -eq 9 ] +then echo "testProcessBlockMerge2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge2_cfg.py &> testProcessBlockMerge2.log || die "cmsRun testProcessBlockMerge2_cfg.py" $? +fi +if [ $1 -eq 10 ] +then echo "testProcessBlockMergeOfMergedFiles" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMergeOfMergedFiles_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles_cfg.py &> testProcessBlockMergeOfMergedFiles.log || die "cmsRun testProcessBlockMergeOfMergedFiles_cfg.py" $? +fi +if [ $1 -eq 11 ] +then echo "testProcessBlockNOMergeOfMergedFiles" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNOMergeOfMergedFiles_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNOMergeOfMergedFiles_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNOMergeOfMergedFiles_cfg.py &> testProcessBlockNOMergeOfMergedFiles.log || die "cmsRun testProcessBlockNOMergeOfMergedFiles_cfg.py" $? +fi +if [ $1 -eq 12 ] +then echo "testProcessBlockRead2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockRead2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockRead2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockRead2_cfg.py &> testProcessBlockRead2.log || die "cmsRun testProcessBlockRead2_cfg.py" $? +fi +if [ $1 -eq 14 ] +then echo "testProcessBlockSubProcess" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcess_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcess_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcess_cfg.py &> testProcessBlockSubProcess.log || die "cmsRun testProcessBlockSubProcess_cfg.py" $? +fi +if [ $1 -eq 15 ] +then echo "testProcessBlockSubProcessRead1" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead1_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessRead1_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead1_cfg.py &> testProcessBlockSubProcessRead1.log || die "cmsRun testProcessBlockSubProcessRead1_cfg.py" $? +fi +if [ $1 -eq 16 ] +then echo "testProcessBlockSubProcessRead2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessRead2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessRead2_cfg.py &> testProcessBlockSubProcessRead2.log || die "cmsRun testProcessBlockSubProcessRead2_cfg.py" $? +fi +if [ $1 -eq 17 ] +then echo "testProcessBlockSubProcessLooper" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessLooper_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockSubProcessLooper_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockSubProcessLooper_cfg.py &> testProcessBlockSubProcessLooper.log || die "cmsRun testProcessBlockSubProcessLooper_cfg.py" $? +fi +if [ $1 -eq 18 ] +then echo "testProcessBlock5" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock5_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlock5_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlock5_cfg.py &> testProcessBlock5.log || die "cmsRun testProcessBlock5_cfg.py" $? echo "testProcessBlockMerge3" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMerge3_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMerge3_cfg.py &> testProcessBlockMerge3.log || die "cmsRun testProcessBlockMerge3_cfg.py" $? echo "testProcessBlockMergeOfMergedFiles2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockMergeOfMergedFiles2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles2_cfg.py &> testProcessBlockMergeOfMergedFiles2.log || die "cmsRun testProcessBlockMergeOfMergedFiles2_cfg.py" $? +fi +if [ $1 -eq 19 ] +then echo "testProcessBlockDropOnInput" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnInput_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnInput_cfg.py &> testProcessBlockDropOnInput.log || die "cmsRun testProcessBlockDropOnInput_cfg.py" $? +fi +if [ $1 -eq 20 ] +then echo "testProcessBlockThreeFileInput" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockThreeFileInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockThreeFileInput_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockThreeFileInput_cfg.py &> testProcessBlockThreeFileInput.log || die "cmsRun testProcessBlockThreeFileInput_cfg.py" $? echo "testProcessBlockReadThreeFileInput" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadThreeFileInput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadThreeFileInput_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadThreeFileInput_cfg.py &> testProcessBlockReadThreeFileInput.log || die "cmsRun testProcessBlockReadThreeFileInput_cfg.py" $? +fi +if [ $1 -eq 21 ] +then echo "testLooperEventNavigation2" - cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation2_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation2.txt > /dev/null 2>&1 || die "cmsRun testLooperEventNavigation2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation2_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation2.txt &> testLooperEventNavigation2.log || die "cmsRun testLooperEventNavigation2_cfg.py" $? +fi +if [ $1 -eq 22 ] +then echo "testLooperEventNavigation3" - cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation3_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation3.txt > /dev/null 2>&1 || die "cmsRun testLooperEventNavigation3_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testLooperEventNavigation3_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation3.txt &> testLooperEventNavigation3.log || die "cmsRun testLooperEventNavigation3_cfg.py" $? +fi +if [ $1 -eq 23 ] +then echo "testProcessBlockDropOnOutput" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnOutput_cfg.py" $? - - echo "testProcessBlockDropOnOutput2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockDropOnOutput2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput_cfg.py &> testProcessBlockDropOnOutput.log || die "cmsRun testProcessBlockDropOnOutput_cfg.py" $? echo "testProcessBlockReadDropOnOutput" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadDropOnOutput_cfg.py" $? - - echo "testProcessBlockReadDropOnOutput2" - cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockReadDropOnOutput2_cfg.py" $? + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput_cfg.py &> testProcessBlockReadDropOnOutput.log || die "cmsRun testProcessBlockReadDropOnOutput_cfg.py" $? +fi - # The next three tests would be relevant if we disabled the strict merging requirement - # in ProductRegistry.cc for ProcessBlock products (a one line code change). As long - # as we always enforce the strict merging requirement these tests will fail, but they - # would be useful if we decide to allow that requirement to be disabled in the future. - # I ran them manually with the ProductRegistry.cc modified to disable the requirement - # and in May 2021 these tests passed. - - #echo "testProcessBlockNonStrict" - #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict_cfg.py" $? - - #echo "testProcessBlockNonStrict2" - #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict2_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict2_cfg.py" $? - - #echo "testProcessBlockNonStrict3" - #cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict3_cfg.py > /dev/null 2>&1 || die "cmsRun testProcessBlockNonStrict3_cfg.py" $? +if [ $1 -eq 24 ] +then + echo "testProcessBlockDropOnOutput2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput2_cfg.py &> testProcessBlockDropOnOutput2.log || die "cmsRun testProcessBlockDropOnOutput2_cfg.py" $? + echo "testProcessBlockReadDropOnOutput2" + cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput2_cfg.py &> testProcessBlockReadDropOnOutput2.log || die "cmsRun testProcessBlockReadDropOnOutput2_cfg.py" $? +fi + +# The next three tests would be relevant if we disabled the strict merging requirement +# in ProductRegistry.cc for ProcessBlock products (a one line code change). As long +# as we always enforce the strict merging requirement these tests will fail, but they +# would be useful if we decide to allow that requirement to be disabled in the future. +# I ran them manually with the ProductRegistry.cc modified to disable the requirement +# and in May 2021 these tests passed. In addition to uncommenting the tests here, they +# would also need to be added in the BuildFile with the proper dependency (both 25 +# and 26 depend on 19 at the moment) + +#if [ $1 -eq 25 ] +#then +# echo "testProcessBlockNonStrict" +# cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict_cfg.py &> testProcessBlockNonStrict.log || die "cmsRun testProcessBlockNonStrict_cfg.py" $? +# +# echo "testProcessBlockNonStrict2" +# cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict2_cfg.py &> testProcessBlockNonStrict2.log || die "cmsRun testProcessBlockNonStrict2_cfg.py" $? +#fi + +#if [ $1 -eq 26 ] +#then +# echo "testProcessBlockNonStrict3" +# cmsRun -p ${LOCAL_TEST_DIR}/testProcessBlockNonStrict3_cfg.py &> testProcessBlockNonStrict3.log || die "cmsRun testProcessBlockNonStrict3_cfg.py" $? +#fi + +if [ $1 -eq 100 ] +then rm testProcessBlock1ContentsM.txt rm testProcessBlock1ContentsE.txt - rm testProcessBlock2DroppedContentsM.txt - rm testProcessBlock2DroppedContentsE.txt rm testProcessBlockMContentsM.txt rm testProcessBlockMContentsE.txt + rm testProcessBlock2DroppedContentsM.txt + rm testProcessBlock2DroppedContentsE.txt + + rm testProcessBlock1.log + rm testProcessBlock2.log + rm testProcessBlock3.log + rm testProcessBlock4.log + rm testProcessBlockMerge.log + rm testProcessBlockTEST.log rm testProcessBlockRead.log + rm testProcessBlock2Dropped.log + rm testProcessBlockFailMerge.log + rm testProcessBlockMerge2.log + rm testProcessBlockMergeOfMergedFiles.log + rm testProcessBlockNOMergeOfMergedFiles.log + rm testProcessBlockRead2.log + rm testProcessBlockSubProcess.log + rm testProcessBlockSubProcessRead1.log + rm testProcessBlockSubProcessRead2.log + rm testProcessBlockSubProcessLooper.log + rm testProcessBlock5.log + rm testProcessBlockMerge3.log + rm testProcessBlockMergeOfMergedFiles2.log + rm testProcessBlockDropOnInput.log + rm testProcessBlockThreeFileInput.log + rm testProcessBlockReadThreeFileInput.log + rm testLooperEventNavigation2.log + rm testLooperEventNavigation3.log + rm testProcessBlockDropOnOutput.log + rm testProcessBlockReadDropOnOutput.log + rm testProcessBlockDropOnOutput2.log + rm testProcessBlockReadDropOnOutput2.log rm testProcessBlock1.root rm testProcessBlock2.root @@ -177,6 +302,23 @@ pushd ${LOCAL_TMP_DIR} rm testProcessBlockDropOnOutput2_2.root rm testProcessBlockReadDropOnOutput.root rm testProcessBlockReadDropOnOutput2.root + rm testProcessBlockNOMergeOfMergedFiles001.root + rm testProcessBlockSubProcessLooperRead001.root + rm testProcessBlockSubProcessLooperRead002.root + rm testProcessBlockSubProcessLooperReadAgain001.root + rm testProcessBlockSubProcessLooperReadAgain002.root + rm testProcessBlockSubProcessLooperTest001.root + rm testProcessBlockSubProcessLooperTest002.root + + #rm testProcessBlockNonStrict.log + #rm testProcessBlockNonStrict2.log + #rm testProcessBlockNonStrict3.log + #rm testProcessBlockNonStrict.root + #rm testProcessBlockNonStrict2.root + #rm testProcessBlockNonStrict3.root + +fi + popd exit 0 diff --git a/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py index 75f07da947fb2..7e9a26e209e8d 100644 --- a/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py +++ b/FWCore/Integration/test/testProcessBlockNonStrict3_cfg.py @@ -56,7 +56,7 @@ ) process.out = cms.OutputModule("PoolOutputModule", - fileName = cms.untracked.string('testProcessBlockNonStrict.root') + fileName = cms.untracked.string('testProcessBlockNonStrict3.root') ) process.testOneOutput = cms.OutputModule("TestOneOutput",