Add enforceGUIDInFileName option to PoolSource and EmbeddedRootSource

FWCore/Framework/python/test/cmsExceptionsFatalOption_cff.py

@@ -30,5 +30,6 @@
   'ProductDoesNotSupportViews',
   'ProductDoesNotSupportPtr',
   'NotFound',
-  'FormatIncompatibility'
+  'FormatIncompatibility',
+  'FileNameInconsistentWithGUID',
 )

FWCore/Utilities/interface/EDMException.h

@@ -16,8 +16,7 @@ namespace edm {
   namespace errors {
 
     // If you add a new entry to the set of values, make sure to
-    // update the translation map in EDMException.cc, the actions
-    // table in FWCore/Framework/src/Actions.cc, and the configuration
+    // update the translation map in EDMException.cc, and the configuration
     // fragment FWCore/Framework/python/test/cmsExceptionsFatalOption_cff.py.
 
     enum ErrorCodes {
@@ -66,6 +65,8 @@ namespace edm {
 
       FileWriteError = 8033,
 
+      FileNameInconsistentWithGUID = 8034,
+
       EventGenerationFailure = 8501,
 
       CaughtSignal = 9000

FWCore/Utilities/src/EDMException.cc

@@ -42,6 +42,7 @@ namespace edm {
                                                               FILLENTRY(ExceededResourceRSS),
                                                               FILLENTRY(ExceededResourceTime),
                                                               FILLENTRY(FileWriteError),
+                                                              FILLENTRY(FileNameInconsistentWithGUID),
                                                               FILLENTRY(EventGenerationFailure),
                                                               FILLENTRY(CaughtSignal)};
     static const std::string kUnknownCode("unknownCode");

IOPool/Input/src/RootEmbeddedFileSequence.cc

@@ -42,7 +42,8 @@ namespace edm {
         // and should be deleted from the code.
         initialNumberOfEventsToSkip_(pset.getUntrackedParameter<unsigned int>("skipEvents", 0U)),
         treeCacheSize_(pset.getUntrackedParameter<unsigned int>("cacheSize", roottree::defaultCacheSize)),
-        enablePrefetching_(false) {
+        enablePrefetching_(false),
+        enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName", false)) {
     if (noFiles()) {
       throw Exception(errors::NoSecondaryFiles)
           << "RootEmbeddedFileSequence no input files specified for secondary input source.\n";
@@ -138,7 +139,8 @@ namespace edm {
                                       currentIndexIntoFile,
                                       orderedProcessHistoryIDs_,
                                       input_.bypassVersionCheck(),
-                                      enablePrefetching_);
+                                      enablePrefetching_,
+                                      enforceGUIDInFileName_);
   }
 
   void RootEmbeddedFileSequence::skipEntries(unsigned int offset) {
@@ -336,5 +338,9 @@ namespace edm {
             "Skip the first 'skipEvents' events. Used only if 'sequential' is True and 'sameLumiBlock' is False");
     desc.addUntracked<unsigned int>("cacheSize", roottree::defaultCacheSize)
         ->setComment("Size of ROOT TTree prefetch cache.  Affects performance.");
+    desc.addUntracked<bool>("enforceGUIDInFileName", false)
+        ->setComment(
+            "True:  file name part is required to be equal to the GUID of the file\n"
+            "False: file name can be anything");
   }
 }  // namespace edm

IOPool/Input/src/RootEmbeddedFileSequence.h

@@ -70,6 +70,7 @@ namespace edm {
     int initialNumberOfEventsToSkip_;
     unsigned int treeCacheSize_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   };  // class RootEmbeddedFileSequence
 }  // namespace edm
 #endif

IOPool/Input/src/RootFile.cc

@@ -164,7 +164,8 @@ namespace edm {
                      bool bypassVersionCheck,
                      bool labelRawDataLikeMC,
                      bool usingGoToEvent,
-                     bool enablePrefetching)
+                     bool enablePrefetching,
+                     bool enforceGUIDInFileName)
       : file_(fileName),
         logicalFile_(logicalFileName),
         processConfiguration_(processConfiguration),
@@ -187,6 +188,7 @@ namespace edm {
         savedRunAuxiliary_(),
         skipAnyEvents_(skipAnyEvents),
         noEventSort_(noEventSort),
+        enforceGUIDInFileName_(enforceGUIDInFileName),
         whyNotFastClonable_(0),
         hasNewlyDroppedBranch_(),
         branchListIndexesUnchanged_(false),
@@ -1139,6 +1141,27 @@ namespace edm {
       throw Exception(errors::EventCorruption) << "'Events' tree is corrupted or not present\n"
                                                << "in the input file.\n";
     }
+    if (enforceGUIDInFileName_) {
+      auto begin = file_.rfind("/");
+      if (begin == std::string::npos) {
+        begin = file_.rfind(":");
+        if (begin == std::string::npos) {
+          // shouldn't really happen?
+          begin = 0;
+        } else {
+          begin += 1;
+        }
+      } else {
+        begin += 1;
+      }
+      auto end = file_.find(".", begin);
+      auto guidFromName = std::string_view(file_).substr(begin, end - begin);
+      if (guidFromName != fid_.fid()) {
+        throw edm::Exception(edm::errors::FileNameInconsistentWithGUID)
+            << "GUID " << guidFromName << " extracted from file name " << file_
+            << " is inconsistent with the GUID read from the file " << fid_.fid();
+      }
+    }
 
     if (fileFormatVersion().hasIndexIntoFile()) {
       if (runTree().entries() > 0) {

IOPool/Input/src/RootFile.h

@@ -91,7 +91,8 @@ namespace edm {
              bool bypassVersionCheck,
              bool labelRawDataLikeMC,
              bool usingGoToEvent,
-             bool enablePrefetching);
+             bool enablePrefetching,
+             bool enforceGUIDInFileName);
 
     RootFile(std::string const& fileName,
              ProcessConfiguration const& processConfiguration,
@@ -113,7 +114,8 @@ namespace edm {
              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
              bool bypassVersionCheck,
              bool labelRawDataLikeMC,
-             bool enablePrefetching)
+             bool enablePrefetching,
+             bool enforceGUIDInFileName)
         : RootFile(fileName,
                    processConfiguration,
                    logicalFileName,
@@ -142,7 +144,8 @@ namespace edm {
                    bypassVersionCheck,
                    labelRawDataLikeMC,
                    false,
-                   enablePrefetching) {}
+                   enablePrefetching,
+                   enforceGUIDInFileName) {}
 
     RootFile(std::string const& fileName,
              ProcessConfiguration const& processConfiguration,
@@ -159,7 +162,8 @@ namespace edm {
              std::vector<std::shared_ptr<IndexIntoFile>>::size_type currentIndexIntoFile,
              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
              bool bypassVersionCheck,
-             bool enablePrefetching)
+             bool enablePrefetching,
+             bool enforceGUIDInFileName)
         : RootFile(fileName,
                    processConfiguration,
                    logicalFileName,
@@ -188,7 +192,8 @@ namespace edm {
                    bypassVersionCheck,
                    false,
                    false,
-                   enablePrefetching) {}
+                   enablePrefetching,
+                   enforceGUIDInFileName) {}
 
     ~RootFile();
 
@@ -325,6 +330,7 @@ namespace edm {
     edm::propagate_const<std::shared_ptr<RunAuxiliary>> savedRunAuxiliary_;
     bool skipAnyEvents_;
     bool noEventSort_;
+    bool enforceGUIDInFileName_;
     int whyNotFastClonable_;
     std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
     bool branchListIndexesUnchanged_;

IOPool/Input/src/RootPrimaryFileSequence.cc

@@ -32,7 +32,8 @@ namespace edm {
         treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter<unsigned int>("cacheSize") : 0U),
         duplicateChecker_(new DuplicateChecker(pset)),
         usingGoToEvent_(false),
-        enablePrefetching_(false) {
+        enablePrefetching_(false),
+        enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName")) {
     // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
     Service<SiteLocalConfig> pSLC;
     if (pSLC.isAvailable()) {
@@ -137,7 +138,8 @@ namespace edm {
                                       input_.bypassVersionCheck(),
                                       input_.labelRawDataLikeMC(),
                                       usingGoToEvent_,
-                                      enablePrefetching_);
+                                      enablePrefetching_,
+                                      enforceGUIDInFileName_);
   }
 
   bool RootPrimaryFileSequence::nextFile() {
@@ -322,6 +324,10 @@ namespace edm {
         ->setComment(
             "'strict':     Branches in each input file must match those in the first file.\n"
             "'permissive': Branches in each input file may be any subset of those in the first file.");
+    desc.addUntracked<bool>("enforceGUIDInFileName", false)
+        ->setComment(
+            "True:  file name part is required to be equal to the GUID of the file\n"
+            "False: file name can be anything");
 
     EventSkipperByID::fillDescription(desc);
     DuplicateChecker::fillDescription(desc);

IOPool/Input/src/RootPrimaryFileSequence.h

@@ -77,6 +77,7 @@ namespace edm {
     edm::propagate_const<std::shared_ptr<DuplicateChecker>> duplicateChecker_;
     bool usingGoToEvent_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   };  // class RootPrimaryFileSequence
 }  // namespace edm
 #endif

IOPool/Input/src/RootSecondaryFileSequence.cc

@@ -21,7 +21,11 @@ namespace edm {
   RootSecondaryFileSequence::RootSecondaryFileSequence(ParameterSet const& pset,
                                                        PoolSource& input,
                                                        InputFileCatalog const& catalog)
-      : RootInputFileSequence(pset, catalog), input_(input), orderedProcessHistoryIDs_(), enablePrefetching_(false) {
+      : RootInputFileSequence(pset, catalog),
+        input_(input),
+        orderedProcessHistoryIDs_(),
+        enablePrefetching_(false),
+        enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName")) {
     // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
     Service<SiteLocalConfig> pSLC;
     if (pSLC.isAvailable()) {
@@ -85,7 +89,8 @@ namespace edm {
                                       orderedProcessHistoryIDs_,
                                       input_.bypassVersionCheck(),
                                       input_.labelRawDataLikeMC(),
-                                      enablePrefetching_);
+                                      enablePrefetching_,
+                                      enforceGUIDInFileName_);
   }
 
   void RootSecondaryFileSequence::initAssociationsFromSecondary(std::set<BranchID> const& associationsFromSecondary) {

IOPool/Input/src/RootSecondaryFileSequence.h

@@ -45,6 +45,7 @@ namespace edm {
     std::vector<BranchID> associationsFromSecondary_;
     std::vector<ProcessHistoryID> orderedProcessHistoryIDs_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   };  // class RootSecondaryFileSequence
 }  // namespace edm
 #endif

IOPool/Input/test/PoolGUIDTest_cfg.py

@@ -0,0 +1,31 @@
+# Configuration file for PoolInputTest
+
+import FWCore.ParameterSet.Config as cms
+import sys
+
+argv = []
+foundpy = False
+for a in sys.argv:
+    if foundpy:
+        argv.append(a)
+    if ".py" in a:
+        foundpy = True
+
+process = cms.Process("TESTRECO")
+process.load("FWCore.Framework.test.cmsExceptionsFatal_cff")
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(-1)
+)
+process.OtherThing = cms.EDProducer("OtherThingProducer")
+process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer")
+
+process.source = cms.Source("PoolSource",
+    setRunNumber = cms.untracked.uint32(621),
+    fileNames = cms.untracked.vstring(argv[0]),
+    enforceGUIDInFileName = cms.untracked.bool(True)
+)
+
+process.p = cms.Path(process.OtherThing*process.Analysis)
+
+

IOPool/Input/test/PrePoolInputTest_cfg.py

@@ -3,26 +3,34 @@
 # Configuration file for PrePoolInputTest 
 
 import FWCore.ParameterSet.Config as cms
-from sys import argv
+import sys
+
+argv = []
+foundpy = False
+for a in sys.argv:
+    if foundpy:
+        argv.append(a)
+    if ".py" in a:
+        foundpy = True
 
 process = cms.Process("TESTPROD")
 process.load("FWCore.Framework.test.cmsExceptionsFatal_cff")
 
 process.maxEvents = cms.untracked.PSet(
-    input = cms.untracked.int32(int(argv[3]))
+    input = cms.untracked.int32(int(argv[1]))
 )
 
 process.Thing = cms.EDProducer("ThingProducer")
 
 process.output = cms.OutputModule("PoolOutputModule",
-    fileName = cms.untracked.string(argv[2])
+    fileName = cms.untracked.string(argv[0])
 )
 
 process.source = cms.Source("EmptySource",
-    firstRun = cms.untracked.uint32(int(argv[4])),
-    numberEventsInRun = cms.untracked.uint32(int(argv[5])),
-    firstLuminosityBlock = cms.untracked.uint32(int(argv[6])),
-    numberEventsInLuminosityBlock = cms.untracked.uint32(int(argv[7]))
+    firstRun = cms.untracked.uint32(int(argv[2])),
+    numberEventsInRun = cms.untracked.uint32(int(argv[3])),
+    firstLuminosityBlock = cms.untracked.uint32(int(argv[4])),
+    numberEventsInLuminosityBlock = cms.untracked.uint32(int(argv[5]))
 )
 
 process.p = cms.Path(process.Thing)

IOPool/Input/test/TestPoolInput.sh

@@ -4,7 +4,13 @@ function die { echo $1: status $2 ;  exit $2; }
 
 pushd ${LOCAL_TMP_DIR}
 
-cmsRun ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py PoolInputTest.root 11 561 7 6 3 || die 'Failure using PrePoolInputTest_cfg.py' $?
+cmsRun -j PoolInputTest_jobreport.xml ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py PoolInputTest.root 11 561 7 6 3 || die 'Failure using PrePoolInputTest_cfg.py' $?
+
+cmsRun ${LOCAL_TEST_DIR}/PoolGUIDTest_cfg.py file:PoolInputTest.root && die 'PoolGUIDTest_cfg.py PoolInputTest.root did not throw an exception' 1
+GUID_NAME=$(fgrep GUID PoolInputTest_jobreport.xml | sed -E 's/<.?GUID>//g').root
+cp PoolInputTest.root ${GUID_NAME}
+cmsRun ${LOCAL_TEST_DIR}/PoolGUIDTest_cfg.py file:${GUID_NAME} || die 'Failure using PoolGUIDTest_cfg.py ${GUID_NAME}' $?
+
 
 cp PoolInputTest.root PoolInputOther.root