Merge pull request #38983 from Dr15Jones/updateDeleteEarly

Modified data product delete early system

Configuration/StandardSequences/python/earlyDeleteSettings_cff.py

@@ -9,58 +9,27 @@
 from RecoTracker.Configuration.customiseEarlyDeleteForCKF import customiseEarlyDeleteForCKF
 from CommonTools.ParticleFlow.Isolation.customiseEarlyDeleteForCandIsoDeposits import customiseEarlyDeleteForCandIsoDeposits
 
-def _hasInputTagModuleLabel(process, pset, psetModLabel, moduleLabels, result):
-    for name in pset.parameterNames_():
-        value = getattr(pset,name)
-        if isinstance(value, cms.PSet):
-            _hasInputTagModuleLabel(process, value, psetModLabel, moduleLabels, result)
-        elif isinstance(value, cms.VPSet):
-            for ps in value:
-                _hasInputTagModuleLabel(process, ps, psetModLabel, moduleLabels, result)
-        elif isinstance(value, cms.VInputTag):
-            for t in value:
-                t2 = t
-                if not isinstance(t, cms.InputTag):
-                    t2 = cms.InputTag(t2)
-                for i,moduleLabel in enumerate(moduleLabels):
-                    if result[i]: continue #no need
-                    if t2.getModuleLabel() == moduleLabel:
-                        result[i]=True
-        elif isinstance(value, cms.InputTag):
-            for i,moduleLabel in enumerate(moduleLabels):
-                if result[i]: continue #no need
-                if value.getModuleLabel() == moduleLabel:
-                    result[i]=True
-        elif isinstance(value, cms.string) and name == "refToPSet_":
-            try:
-                ps = getattr(process, value.value())
-            except AttributeError:
-                raise RuntimeError("Module %s has a 'PSet(refToPSet_ = cms.string(\"%s\"))', but the referenced-to PSet does not exist in the Process." % (psetModLabel, value.value()))
-            _hasInputTagModuleLabel(process, ps, psetModLabel, moduleLabels, result)
-
-
 def customiseEarlyDelete(process):
     # Mapping label -> [branches]
     # for the producers whose products are to be deleted early
     products = collections.defaultdict(list)
 
-    products = customiseEarlyDeleteForSeeding(process, products)
+    (products, references) = customiseEarlyDeleteForSeeding(process, products)
     products = customiseEarlyDeleteForMkFit(process, products)
     products = customiseEarlyDeleteForCKF(process, products)
 
     products = customiseEarlyDeleteForCandIsoDeposits(process, products)
 
-    # Set process.options.canDeleteEarly
-    if not hasattr(process.options, "canDeleteEarly"):
-        process.options.canDeleteEarly = cms.untracked.vstring()
-
     branchSet = set()
     for branches in products.values():
         for branch in branches:
             branchSet.add(branch)
     branchList = sorted(branchSet)
     process.options.canDeleteEarly.extend(branchList)
 
+    for prod, refs in references.items():
+        process.options.holdsReferencesToDeleteEarly.append(cms.PSet(product=cms.string(prod), references=cms.vstring(refs)))
+
     # LogErrorHarvester should not wait for deleted items
     for prod in process.producers_().values():
         if prod.type_() == "LogErrorHarvester":
@@ -70,80 +39,4 @@ def customiseEarlyDelete(process):
             t.extend([b.split('_')[1] for b in branchList])
             prod.excludeModules = t
 
-    # Find the consumers
-    producers=[]
-    branchesList=[]
-    for producer, branches in products.items():
-        producers.append(producer)
-        branchesList.append(branches)
-
-    for moduleType in [process.producers_(), process.filters_(), process.analyzers_()]:
-        for name, module in moduleType.items():
-            result=[]
-            for producer in producers:
-                result.append(False)
-
-            _hasInputTagModuleLabel(process, module, name, producers, result)
-            for i in range(len(result)):
-                if result[i]:
-                    #if it exists it might be optional or empty, both evaluate to False
-                    if hasattr(module, "mightGet") and module.mightGet:
-                        module.mightGet.extend(branchesList[i])
-                    else:
-                        module.mightGet = cms.untracked.vstring(branchesList[i])
     return process
-
-
-if __name__=="__main__":
-    import unittest
-
-    class TestHasInputTagModuleLabel(unittest.TestCase):
-        def setUp(self):
-            """Nothing to do """
-            None
-        def testHasInputTagModuleLabel(self):
-            p = cms.Process("A")
-            p.pset = cms.PSet(a=cms.InputTag("a"),a2=cms.untracked.InputTag("a2"))
-            p.prod = cms.EDProducer("Producer",
-                foo = cms.InputTag("foo"),
-                foo2 = cms.InputTag("foo2", "instance"),
-                foo3 = cms.InputTag("foo3", "instance", "PROCESS"),
-                foo4 = cms.untracked.InputTag("foo4"),
-                nested = cms.PSet(
-                    bar = cms.InputTag("bar"),
-                    bar2 = cms.untracked.InputTag("bar2"),
-                ),
-                nested2 = cms.untracked.PSet(
-                    bar3 = cms.untracked.InputTag("bar3"),
-                ),
-                flintstones = cms.VPSet(
-                    cms.PSet(fred=cms.InputTag("fred")),
-                    cms.PSet(wilma=cms.InputTag("wilma"))
-                ),
-                flintstones2 = cms.VPSet(
-                    cms.PSet(fred2=cms.untracked.InputTag("fred2")),
-                    cms.PSet(wilma2=cms.InputTag("wilma2"))
-                ),
-                ref = cms.PSet(
-                    refToPSet_ = cms.string("pset")
-                ),
-                ref2 = cms.untracked.PSet(
-                    refToPSet_ = cms.string("pset")
-                ),
-            )
-            p.prod2 = cms.EDProducer("Producer2",
-                foo = cms.PSet(
-                    refToPSet_ = cms.string("nonexistent")
-                )
-            )
-
-            result=[False,False,False,False,False,False,False,False,False,False,False,False,False,False]
-            _hasInputTagModuleLabel(p, p.prod, "prod", ["foo","foo2","foo3","bar","fred","wilma","a","foo4","bar2","bar3","fred2","wilma2","a2","joe"], result)
-            for i in range (0,13):
-                self.assert_(result[i])
-            self.assert_(not result[13])
-
-            result = [False]
-            self.assertRaises(RuntimeError, _hasInputTagModuleLabel, p, p.prod2, "prod2", ["foo"], result)
-
-    unittest.main()

DataFormats/Common/interface/Wrapper.h

@@ -25,7 +25,7 @@ namespace edm {
   public:
     typedef T value_type;
     typedef T wrapped_type;  // used with the dictionary to identify Wrappers
-    Wrapper() : WrapperBase(), present(false), obj() {}
+    Wrapper() : WrapperBase(), obj(), present(false) {}
     explicit Wrapper(std::unique_ptr<T> ptr);
     Wrapper(Wrapper<T> const& rh) = delete;             // disallow copy construction
     Wrapper<T>& operator=(Wrapper<T> const&) = delete;  // disallow assignment
@@ -46,7 +46,7 @@ namespace edm {
     Wrapper(T*);
 
     //Used by ROOT storage
-    CMS_CLASS_VERSION(3)
+    CMS_CLASS_VERSION(4)
 
   private:
     bool isPresent_() const override { return present; }
@@ -73,20 +73,20 @@ namespace edm {
     std::shared_ptr<soa::TableExaminerBase> tableExaminer_() const override;
 
   private:
-    bool present;
     T obj;
+    bool present;
   };
 
   template <typename T>
-  Wrapper<T>::Wrapper(std::unique_ptr<T> ptr) : WrapperBase(), present(ptr.get() != nullptr), obj() {
+  Wrapper<T>::Wrapper(std::unique_ptr<T> ptr) : WrapperBase(), obj(), present(ptr.get() != nullptr) {
     if (present) {
       obj = std::move(*ptr);
     }
   }
 
   template <typename T>
   template <typename... Args>
-  Wrapper<T>::Wrapper(Emplace, Args&&... args) : WrapperBase(), present(true), obj(std::forward<Args>(args)...) {}
+  Wrapper<T>::Wrapper(Emplace, Args&&... args) : WrapperBase(), obj(std::forward<Args>(args)...), present(true) {}
 
   template <typename T>
   Wrapper<T>::Wrapper(T* ptr) : WrapperBase(), present(ptr != 0), obj() {

DataFormats/TestObjects/src/classes.h

@@ -29,6 +29,7 @@
 #include "DataFormats/TestObjects/interface/DeleteEarly.h"
 
 #include "DataFormats/Common/interface/Holder.h"
+#include "DataFormats/Common/interface/RefProd.h"
 #include "DataFormats/Common/interface/RefToBaseProd.h"
 #include "DataFormats/Common/interface/RefToBaseVector.h"
 #include "DataFormats/Common/interface/RandomNumberGeneratorState.h"

DataFormats/TestObjects/src/classes_def.xml

@@ -168,7 +168,9 @@
    <version ClassVersion="3" checksum="1338867524"/>
  </class>
  <class name="edm::Wrapper<edmtest::DeleteEarly>"/>
-
+ <class name="edm::RefProd<edmtest::DeleteEarly>"/>
+ <class name="edm::Wrapper<edm::RefProd<edmtest::DeleteEarly>>"/>
+
  <class name="edm::helpers::KeyVal<edm::RefProd<std::vector<int> >,edm::RefProd<std::vector<int> > >"/>
  <class name="edm::AssociationMap<edm::OneToOne<std::vector<int>,std::vector<int>,unsigned int> >">
     <field name="transientMap_" transient="true" />

FWCore/Framework/interface/EventProcessor.h

@@ -332,6 +332,8 @@ namespace edm {
     std::atomic<unsigned int> streamLumiActive_{0};  //works as guard for streamLumiStatus
 
     std::vector<std::string> branchesToDeleteEarly_;
+    std::multimap<std::string, std::string> referencesToBranches_;
+    std::vector<std::string> modulesToIgnoreForDeleteEarly_;
 
     std::vector<SubProcess> subProcesses_;
     edm::propagate_const<std::unique_ptr<HistoryAppender>> historyAppender_;

FWCore/Framework/interface/Schedule.h

@@ -285,7 +285,10 @@ namespace edm {
     /// Deletes module with label iLabel
     void deleteModule(std::string const& iLabel, ActivityRegistry* areg);
 
-    void initializeEarlyDelete(std::vector<std::string> const& branchesToDeleteEarly, edm::ProductRegistry const& preg);
+    void initializeEarlyDelete(std::vector<std::string> const& branchesToDeleteEarly,
+                               std::multimap<std::string, std::string> const& referencesToBranches,
+                               std::vector<std::string> const& modulesToSkip,
+                               edm::ProductRegistry const& preg);
 
     /// returns the collection of pointers to workers
     AllWorkers const& allWorkers() const;

FWCore/Framework/interface/StreamSchedule.h

@@ -249,6 +249,8 @@ namespace edm {
 
     void initializeEarlyDelete(ModuleRegistry& modReg,
                                std::vector<std::string> const& branchesToDeleteEarly,
+                               std::multimap<std::string, std::string> const& referencesToBranches,
+                               std::vector<std::string> const& modulesToSkip,
                                edm::ProductRegistry const& preg);
 
     /// returns the collection of pointers to workers

FWCore/Framework/src/EventProcessor.cc

@@ -448,6 +448,19 @@ namespace edm {
     if (not hasSubProcesses) {
       branchesToDeleteEarly_ = optionsPset.getUntrackedParameter<std::vector<std::string>>("canDeleteEarly");
     }
+    if (not branchesToDeleteEarly_.empty()) {
+      auto referencePSets =
+          optionsPset.getUntrackedParameter<std::vector<edm::ParameterSet>>("holdsReferencesToDeleteEarly");
+      for (auto const& pset : referencePSets) {
+        auto product = pset.getParameter<std::string>("product");
+        auto references = pset.getParameter<std::vector<std::string>>("references");
+        for (auto const& ref : references) {
+          referencesToBranches_.emplace(product, ref);
+        }
+      }
+      modulesToIgnoreForDeleteEarly_ =
+          optionsPset.getUntrackedParameter<std::vector<std::string>>("modulesToIgnoreForDeleteEarly");
+    }
 
     // Now do general initialization
     ScheduleItems items;
@@ -695,8 +708,10 @@ namespace edm {
     // modules to avoid non-consumed non-run modules to keep the
     // products unnecessarily alive
     if (not branchesToDeleteEarly_.empty()) {
-      schedule_->initializeEarlyDelete(branchesToDeleteEarly_, *preg_);
-      decltype(branchesToDeleteEarly_)().swap(branchesToDeleteEarly_);
+      auto modulesToSkip = std::move(modulesToIgnoreForDeleteEarly_);
+      auto branchesToDeleteEarly = std::move(branchesToDeleteEarly_);
+      auto referencesToBranches = std::move(referencesToBranches_);
+      schedule_->initializeEarlyDelete(branchesToDeleteEarly, referencesToBranches, modulesToSkip, *preg_);
     }
 
     actReg_->preBeginJobSignal_(pathsAndConsumesOfModules_, processContext_);

FWCore/Framework/src/Schedule.cc

@@ -1256,9 +1256,12 @@ namespace edm {
   }
 
   void Schedule::initializeEarlyDelete(std::vector<std::string> const& branchesToDeleteEarly,
+                                       std::multimap<std::string, std::string> const& referencesToBranches,
+                                       std::vector<std::string> const& modulesToSkip,
                                        edm::ProductRegistry const& preg) {
     for (auto& stream : streamSchedules_) {
-      stream->initializeEarlyDelete(*moduleRegistry(), branchesToDeleteEarly, preg);
+      stream->initializeEarlyDelete(
+          *moduleRegistry(), branchesToDeleteEarly, referencesToBranches, modulesToSkip, preg);
     }
   }