Merge pull request #28597 from makortel/enforceGUIDInFileName_9_4_x

Add enforceGUIDInFileName option to PoolSource and EmbeddedRootSource (9_4_X)

FWCore/Framework/python/test/cmsExceptionsFatalOption_cff.py

@@ -30,5 +30,6 @@
   'ProductDoesNotSupportViews',
   'ProductDoesNotSupportPtr',
   'NotFound',
-  'FormatIncompatibility'
+  'FormatIncompatibility',
+  'FileNameInconsistentWithGUID',
 )

FWCore/MessageLogger/scripts/edmFjrDump

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+import xml.etree.ElementTree as ET
+import argparse
+
+def printGUID(root):
+    for f in root.findall("File"):
+        print(f.find("GUID").text)
+
+def printExitCode(root):
+    error = root.find("FrameworkError")
+    if error is None:
+        print("0")
+        return
+    print(error.get("ExitStatus"))
+
+def main(opts):
+    tree = ET.parse(opts.file)
+    root = tree.getroot()
+    if opts.guid:
+        printGUID(root)
+    if opts.exitCode:
+        printExitCode(root)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Extract some values from the Framework Job Report")
+    parser.add_argument("file", type=str,
+                        help="Framework Job Report XML file")
+    parser.add_argument("--guid", action="store_true",
+                        help="GUID of the output file")
+    parser.add_argument("--exitCode", action="store_true",
+                        help="Job exit code")
+
+    opts = parser.parse_args()
+    main(opts)

FWCore/Utilities/interface/EDMException.h

@@ -16,8 +16,7 @@ namespace edm {
   namespace errors {
 
     // If you add a new entry to the set of values, make sure to
-    // update the translation map in EDMException.cc, the actions
-    // table in FWCore/Framework/src/Actions.cc, and the configuration
+    // update the translation map in EDMException.cc, and the configuration
     // fragment FWCore/Framework/python/test/cmsExceptionsFatalOption_cff.py.
 
     enum ErrorCodes {
@@ -63,6 +62,8 @@ namespace edm {
        ExceededResourceRSS = 8031,
        ExceededResourceTime = 8032,
 
+       FileNameInconsistentWithGUID = 8034,
+
        CaughtSignal = 9000
     };
 

FWCore/Utilities/interface/stemFromPath.h

@@ -0,0 +1,17 @@
+#ifndef FWCore_Utilities_stemFromPath_h
+#define FWCore_Utilities_stemFromPath_h
+
+#include <string>
+
+namespace edm {
+  // This functions extracts the stem of a file from the path (= file
+  // name without the extension).
+  //
+  // The reason to have our own function instead of
+  // std/boost::filesystem is that tehcnically these paths are not
+  // filesystem paths, but paths in CMS LFN/PFN space that (may) have
+  // different rules.
+  std::string stemFromPath(const std::string& path);
+}  // namespace edm
+
+#endif

FWCore/Utilities/src/EDMException.cc

@@ -41,6 +41,7 @@ namespace edm {
       FILLENTRY(ExceededResourceVSize),
       FILLENTRY(ExceededResourceRSS),
       FILLENTRY(ExceededResourceTime),
+      FILLENTRY(FileNameInconsistentWithGUID),
       FILLENTRY(CaughtSignal)
     };
     static const std::string kUnknownCode("unknownCode");

FWCore/Utilities/src/stemFromPath.cc

@@ -0,0 +1,20 @@
+#include "FWCore/Utilities/interface/stemFromPath.h"
+
+namespace edm {
+  std::string stemFromPath(const std::string& path) {
+    auto begin = path.rfind("/");
+    if (begin == std::string::npos) {
+      begin = path.rfind(":");
+      if (begin == std::string::npos) {
+        // shouldn't really happen?
+        begin = 0;
+      } else {
+        begin += 1;
+      }
+    } else {
+      begin += 1;
+    }
+    auto end = path.find(".", begin);
+    return path.substr(begin, end - begin);
+  }
+}  // namespace edm

FWCore/Utilities/test/BuildFile.xml

@@ -34,6 +34,8 @@
 <bin   name="testFWCoreUtilities" file="typeidbase_t.cppunit.cpp,typeid_t.cppunit.cpp,cputimer_t.cppunit.cpp,extensioncord_t.cppunit.cpp,friendlyname_t.cppunit.cpp,signal_t.cppunit.cpp,soatuple_t.cppunit.cpp,transform.cppunit.cpp,callxnowait_t.cppunit.cpp,vecarray.cppunit.cpp,reusableobjectholder_t.cppunit.cpp,propagate_const_t.cppunit.cpp,indexset.cppunit.cpp">
   <use   name="cppunit"/>
 </bin>
+<bin    file="test_stemFromPath.cc" name="testFWCoreUtilitiesStem">
+</bin>
 
 <bin   file="InputTag_t.cpp">
   <use name="tbb"/>

FWCore/Utilities/test/test_stemFromPath.cc

@@ -0,0 +1,16 @@
+#include "FWCore/Utilities/interface/stemFromPath.h"
+
+#include <cassert>
+
+int main() {
+  assert(edm::stemFromPath("foo.root") == "foo");
+  assert(edm::stemFromPath("/foo.root") == "foo");
+  assert(edm::stemFromPath("/bar/foo.root") == "foo");
+  assert(edm::stemFromPath("/bar///....//...///foo.root") == "foo");
+  assert(edm::stemFromPath("/bar/foo.xyzzy") == "foo");
+  assert(edm::stemFromPath("/bar/xyzzy.foo.root") == "xyzzy");
+  assert(edm::stemFromPath("file:foo.root") == "foo");
+  assert(edm::stemFromPath("file:/path/to/bar.txt") == "bar");
+  assert(edm::stemFromPath("root://server.somewhere:port/whatever?param=path/to/bar.txt") == "bar");
+  return 0;
+}

IOPool/Input/src/RootEmbeddedFileSequence.cc

@@ -43,7 +43,8 @@ namespace edm {
     // and should be deleted from the code.
     initialNumberOfEventsToSkip_(pset.getUntrackedParameter<unsigned int>("skipEvents", 0U)),
     treeCacheSize_(pset.getUntrackedParameter<unsigned int>("cacheSize", roottree::defaultCacheSize)),
-    enablePrefetching_(false) {
+    enablePrefetching_(false),
+    enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName", false)) {
 
     if(noFiles()) {
       throw Exception(errors::Configuration) << "RootEmbeddedFileSequence no input files specified for secondary input source.\n";
@@ -144,7 +145,8 @@ namespace edm {
           currentIndexIntoFile,
           orderedProcessHistoryIDs_,
           input_.bypassVersionCheck(),
-          enablePrefetching_);
+          enablePrefetching_,
+          enforceGUIDInFileName_);
   }
 
   void
@@ -336,5 +338,9 @@ namespace edm {
         ->setComment("Skip the first 'skipEvents' events. Used only if 'sequential' is True and 'sameLumiBlock' is False");
     desc.addUntracked<unsigned int>("cacheSize", roottree::defaultCacheSize)
         ->setComment("Size of ROOT TTree prefetch cache.  Affects performance.");
+    desc.addUntracked<bool>("enforceGUIDInFileName", false)
+        ->setComment(
+            "True:  file name part is required to be equal to the GUID of the file\n"
+            "False: file name can be anything");
   }
 }

IOPool/Input/src/RootEmbeddedFileSequence.h

@@ -65,6 +65,7 @@ namespace edm {
     int initialNumberOfEventsToSkip_;
     unsigned int treeCacheSize_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   }; // class RootEmbeddedFileSequence
 }
 #endif

IOPool/Input/src/RootFile.cc

@@ -41,6 +41,7 @@
 #include "FWCore/Utilities/interface/FriendlyName.h"
 #include "FWCore/Utilities/interface/GlobalIdentifier.h"
 #include "FWCore/Utilities/interface/ReleaseVersion.h"
+#include "FWCore/Utilities/interface/stemFromPath.h"
 #include "FWCore/Version/interface/GetReleaseVersion.h"
 #include "IOPool/Common/interface/getWrapperBasePtr.h"
 
@@ -155,7 +156,8 @@ namespace edm {
                      bool bypassVersionCheck,
                      bool labelRawDataLikeMC,
                      bool usingGoToEvent,
-                     bool enablePrefetching) :
+                     bool enablePrefetching,
+                     bool enforceGUIDInFileName) :
       file_(fileName),
       logicalFile_(logicalFileName),
       processConfiguration_(processConfiguration),
@@ -175,6 +177,7 @@ namespace edm {
       savedRunAuxiliary_(),
       skipAnyEvents_(skipAnyEvents),
       noEventSort_(noEventSort),
+      enforceGUIDInFileName_(enforceGUIDInFileName),
       whyNotFastClonable_(0),
       hasNewlyDroppedBranch_(),
       branchListIndexesUnchanged_(false),
@@ -1097,6 +1100,14 @@ namespace edm {
       throw Exception(errors::EventCorruption) <<
          "'Events' tree is corrupted or not present\n" << "in the input file.\n";
     }
+    if (enforceGUIDInFileName_) {
+      auto guidFromName = stemFromPath(file_);
+      if (guidFromName != fid_.fid()) {
+        throw edm::Exception(edm::errors::FileNameInconsistentWithGUID)
+            << "GUID " << guidFromName << " extracted from file name " << file_
+            << " is inconsistent with the GUID read from the file " << fid_.fid();
+      }
+    }
 
     if(fileFormatVersion().hasIndexIntoFile()) {
       if(runTree().entries() > 0) {

IOPool/Input/src/RootFile.h

@@ -89,7 +89,8 @@ namespace edm {
              bool bypassVersionCheck,
              bool labelRawDataLikeMC,
              bool usingGoToEvent,
-             bool enablePrefetching);
+             bool enablePrefetching,
+             bool enforceGUIDInFileName);
 
     RootFile(std::string const& fileName,
              ProcessConfiguration const& processConfiguration,
@@ -111,7 +112,8 @@ namespace edm {
              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
              bool bypassVersionCheck,
              bool labelRawDataLikeMC,
-             bool enablePrefetching) : RootFile(
+             bool enablePrefetching,
+             bool enforceGUIDInFileName) : RootFile(
                fileName, processConfiguration, logicalFileName, filePtr,
                nullptr, false, -1, -1, nStreams, 0U, treeMaxVirtualSize,
                processingMode, runHelper,
@@ -120,7 +122,7 @@ namespace edm {
                nullptr, dropDescendantsOfDroppedProducts, processHistoryRegistry,
                indexesIntoFiles, currentIndexIntoFile, orderedProcessHistoryIDs,
                bypassVersionCheck, labelRawDataLikeMC,
-               false, enablePrefetching) {}
+               false, enablePrefetching, enforceGUIDInFileName) {}
 
     RootFile(std::string const& fileName,
              ProcessConfiguration const& processConfiguration,
@@ -137,15 +139,16 @@ namespace edm {
              std::vector<std::shared_ptr<IndexIntoFile> >::size_type currentIndexIntoFile,
              std::vector<ProcessHistoryID>& orderedProcessHistoryIDs,
              bool bypassVersionCheck,
-             bool enablePrefetching) : RootFile(
+             bool enablePrefetching,
+             bool enforceGUIDInFileName) : RootFile(
                fileName, processConfiguration, logicalFileName, filePtr,
                nullptr, false, -1, -1, nStreams, treeCacheSize, treeMaxVirtualSize,
                InputSource::RunsLumisAndEvents, runHelper,
                false, productSelectorRules, inputType, nullptr, nullptr,
                nullptr, nullptr, false, processHistoryRegistry,
                indexesIntoFiles, currentIndexIntoFile, orderedProcessHistoryIDs,
                bypassVersionCheck, false,
-               false, enablePrefetching) {}
+               false, enablePrefetching, enforceGUIDInFileName) {}
 
     ~RootFile();
 
@@ -269,6 +272,7 @@ namespace edm {
     edm::propagate_const<std::shared_ptr<RunAuxiliary>> savedRunAuxiliary_;
     bool skipAnyEvents_;
     bool noEventSort_;
+    bool enforceGUIDInFileName_;
     int whyNotFastClonable_;
     std::array<bool, NumBranchTypes> hasNewlyDroppedBranch_;
     bool branchListIndexesUnchanged_;

IOPool/Input/src/RootPrimaryFileSequence.cc

@@ -33,7 +33,8 @@ namespace edm {
     treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter<unsigned int>("cacheSize") : 0U),
     duplicateChecker_(new DuplicateChecker(pset)),
     usingGoToEvent_(false),
-    enablePrefetching_(false) {
+    enablePrefetching_(false),
+    enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName")) {
 
     // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
     Service<SiteLocalConfig> pSLC;
@@ -143,7 +144,8 @@ namespace edm {
           input_.bypassVersionCheck(),
           input_.labelRawDataLikeMC(),
           usingGoToEvent_,
-          enablePrefetching_);
+          enablePrefetching_,
+          enforceGUIDInFileName_);
   }
 
   bool RootPrimaryFileSequence::nextFile() {
@@ -334,6 +336,10 @@ namespace edm {
     desc.addUntracked<std::string>("branchesMustMatch", defaultString)
         ->setComment("'strict':     Branches in each input file must match those in the first file.\n"
                      "'permissive': Branches in each input file may be any subset of those in the first file.");
+    desc.addUntracked<bool>("enforceGUIDInFileName", false)
+        ->setComment(
+            "True:  file name part is required to be equal to the GUID of the file\n"
+            "False: file name can be anything");
 
     EventSkipperByID::fillDescription(desc);
     DuplicateChecker::fillDescription(desc);

IOPool/Input/src/RootPrimaryFileSequence.h

@@ -78,6 +78,7 @@ namespace edm {
     edm::propagate_const<std::shared_ptr<DuplicateChecker>> duplicateChecker_;
     bool usingGoToEvent_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   }; // class RootPrimaryFileSequence
 }
 #endif

IOPool/Input/src/RootSecondaryFileSequence.cc

@@ -25,7 +25,8 @@ namespace edm {
     RootInputFileSequence(pset, catalog),
     input_(input),
     orderedProcessHistoryIDs_(),
-    enablePrefetching_(false) {
+    enablePrefetching_(false),
+    enforceGUIDInFileName_(pset.getUntrackedParameter<bool>("enforceGUIDInFileName")) {
 
     // The SiteLocalConfig controls the TTreeCache size and the prefetching settings.
     Service<SiteLocalConfig> pSLC;
@@ -95,7 +96,8 @@ namespace edm {
           orderedProcessHistoryIDs_,
           input_.bypassVersionCheck(),
           input_.labelRawDataLikeMC(),
-          enablePrefetching_);
+          enablePrefetching_,
+          enforceGUIDInFileName_);
   }
 
   void

IOPool/Input/src/RootSecondaryFileSequence.h

@@ -46,6 +46,7 @@ namespace edm {
     std::vector<BranchID> associationsFromSecondary_;
     std::vector<ProcessHistoryID> orderedProcessHistoryIDs_;
     bool enablePrefetching_;
+    bool enforceGUIDInFileName_;
   }; // class RootSecondaryFileSequence
 }
 #endif

IOPool/Input/test/PoolGUIDTest_cfg.py

@@ -0,0 +1,31 @@
+# Configuration file for PoolInputTest
+
+import FWCore.ParameterSet.Config as cms
+import sys
+
+argv = []
+foundpy = False
+for a in sys.argv:
+    if foundpy:
+        argv.append(a)
+    if ".py" in a:
+        foundpy = True
+
+process = cms.Process("TESTRECO")
+process.load("FWCore.Framework.test.cmsExceptionsFatal_cff")
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(-1)
+)
+process.OtherThing = cms.EDProducer("OtherThingProducer")
+process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer")
+
+process.source = cms.Source("PoolSource",
+    setRunNumber = cms.untracked.uint32(621),
+    fileNames = cms.untracked.vstring(argv[0]),
+    enforceGUIDInFileName = cms.untracked.bool(True)
+)
+
+process.p = cms.Path(process.OtherThing*process.Analysis)
+
+