New developments using multiple data catalogs provided in site-local-config.xml

DQMServices/FwkIO/plugins/DQMRootSource.cc

@@ -425,7 +425,7 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou
       m_fileMetadatas(std::vector<FileMetadata>()) {
   edm::sortAndRemoveOverlaps(m_lumisToProcess);
 
-  if (m_catalog.fileNames().empty()) {
+  if (m_catalog.fileNames(0).empty()) {
     m_nextItemType = edm::InputSource::IsStop;
   } else {
     m_treeReaders[kIntIndex].reset(new TreeSimpleReader<Long64_t>(MonitorElementData::Kind::INT, m_rescope));
@@ -464,110 +464,80 @@ edm::InputSource::ItemType DQMRootSource::getNextItemType() { return m_nextItemT
 
 // We will read the metadata of all files and fill m_fileMetadatas vector
 std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
-  const int numFiles = m_catalog.fileNames().size();
+  const int numFiles = m_catalog.fileNames(0).size();
   m_openFiles.reserve(numFiles);
 
   for (auto& fileitem : m_catalog.fileCatalogItems()) {
-    const auto& filename = fileitem.fileName();
-    const auto& fallbackname = fileitem.fallbackFileName();
-    bool hasFallback = !fallbackname.empty() && fallbackname != filename;
     TFile* file;
-    std::list<std::string> originalInfo;
-
-    // Try to open a file
-    try {
-      file = TFile::Open(filename.c_str());
-
-      // Exception will be trapped so we pull it out ourselves
-      std::exception_ptr e = edm::threadLocalException::getException();
-      if (e != std::exception_ptr()) {
-        edm::threadLocalException::setException(std::exception_ptr());
-        std::rethrow_exception(e);
-      }
-
-    } catch (cms::Exception const& e) {
-      file = nullptr;  // is there anything we need to free?
-      if (!hasFallback) {
-        if (m_skipBadFiles) {
-          continue;
-        } else {
-          edm::Exception ex(edm::errors::FileOpenError, "", e);
-          ex.addContext("Opening DQM Root file");
-          ex << "\nInput file " << filename << " was not found, could not be opened, or is corrupted.\n";
-          throw ex;
-        }
-      }
-      originalInfo = e.additionalInfo();  // save in case of fallback error
-    }
-
-    // Check if a file is usable
-    if (file && !file->IsZombie()) {
-      logFileAction("Successfully opened file ", filename.c_str());
-    } else {
-      if (!hasFallback) {
-        if (!m_skipBadFiles) {
-          edm::Exception ex(edm::errors::FileOpenError);
-          ex << "Input file " << filename.c_str() << " could not be opened.\n";
-          ex.addContext("Opening DQM Root file");
-          throw ex;
-        }
-      }
-      if (file) {
-        delete file;
-        file = nullptr;
-      }
-    }
-
-    if (!file && hasFallback) {
-      logFileAction("  Initiating request to open fallback file ", fallbackname.c_str());
+    std::list<std::string> exInfo;
+    //loop over names of a file, each of them corresponds to a data catalog
+    bool isGoodFile(true);
+    //get all names of a file, each of them corresponds to a data catalog
+    const std::vector<std::string>& fNames = fileitem.fileNames();
+    for (std::vector<std::string>::const_iterator it = fNames.begin(); it != fNames.end(); ++it) {
+      // Try to open a file
       try {
-        {
-          TDirectory::TContext contextEraser;
-          file = TFile::Open(fallbackname.c_str());
-        }
+        file = TFile::Open(it->c_str());
+
+        // Exception will be trapped so we pull it out ourselves
         std::exception_ptr e = edm::threadLocalException::getException();
         if (e != std::exception_ptr()) {
           edm::threadLocalException::setException(std::exception_ptr());
           std::rethrow_exception(e);
         }
+
       } catch (cms::Exception const& e) {
-        file = nullptr;  // is there anything we need to free?
-        if (m_skipBadFiles) {
-          continue;
-        } else {
-          edm::Exception ex(edm::errors::FileOpenError, "", e);
-          ex.addContext("Opening DQM Root file");
-          ex << "\nInput file " << filename << " and fallback input file " << fallbackname
-             << " were not found, could not be opened, or are corrupted.\n";
-          for (auto const& s : originalInfo) {
-            ex.addAdditionalInfo(s);
+        file = nullptr;                       // is there anything we need to free?
+        if (std::next(it) == fNames.end()) {  //last name corresponding to the last data catalog to try
+          if (!m_skipBadFiles) {
+            edm::Exception ex(edm::errors::FileOpenError, "", e);
+            ex.addContext("Opening DQM Root file");
+            ex << "\nInput file " << it->c_str() << " was not found, could not be opened, or is corrupted.\n";
+            //report previous exceptions when use other names to open file
+            for (auto const& s : exInfo)
+              ex.addAdditionalInfo(s);
+            throw ex;
           }
-          throw ex;
+          isGoodFile = false;
         }
+        // save in case of error when trying next name
+        for (auto const& s : e.additionalInfo())
+          exInfo.push_back(s);
       }
-      if (not file->IsZombie()) {
-        logFileAction("  Successfully opened fallback file ", fallbackname.c_str());
+
+      // Check if a file is usable
+      if (file && !file->IsZombie()) {
+        logFileAction("Successfully opened file ", it->c_str());
+        break;
       } else {
-        if (m_skipBadFiles) {
-          continue;
-        } else {
-          edm::Exception ex(edm::errors::FileOpenError);
-          ex << "Input file " << filename << " and fallback input file " << fallbackname << " could not be opened.\n";
-          ex.addContext("Opening DQM Root file");
-          for (auto const& s : originalInfo) {
-            ex.addAdditionalInfo(s);
+        if (std::next(it) == fNames.end()) {
+          if (!m_skipBadFiles) {
+            edm::Exception ex(edm::errors::FileOpenError);
+            ex << "Input file " << it->c_str() << " could not be opened.\n";
+            ex.addContext("Opening DQM Root file");
+            //report previous exceptions when use other names to open file
+            for (auto const& s : exInfo)
+              ex.addAdditionalInfo(s);
+            throw ex;
           }
-          throw ex;
+          isGoodFile = false;
+        }
+        if (file) {
+          delete file;
+          file = nullptr;
         }
       }
-    }
+    }  //end loop over names of the file
+
+    if (!isGoodFile && m_skipBadFiles)
+      continue;
 
     m_openFiles.insert(m_openFiles.begin(), file);
 
     // Check file format version, which is encoded in the Title of the TFile
     if (strcmp(file->GetTitle(), "1") != 0) {
       edm::Exception ex(edm::errors::FileReadError);
-      ex << "Input file " << filename.c_str() << " does not appear to be a DQM Root file.\n";
+      ex << "Input file " << fNames[0].c_str() << " does not appear to be a DQM Root file.\n";
     }
 
     // Read metadata from the file
@@ -592,7 +562,8 @@ std::unique_ptr<edm::FileBlock> DQMRootSource::readFile_() {
         m_fileMetadatas.push_back(temp);
       }
     }
-  }
+
+  }  //end loop over files
 
   // Sort to make sure runs and lumis appear in sequential order
   std::stable_sort(m_fileMetadatas.begin(), m_fileMetadatas.end());

EventFilter/Utilities/plugins/FRDStreamSource.cc

@@ -21,7 +21,7 @@ FRDStreamSource::FRDStreamSource(edm::ParameterSet const& pset, edm::InputSource
       verifyAdler32_(pset.getUntrackedParameter<bool>("verifyAdler32", true)),
       verifyChecksum_(pset.getUntrackedParameter<bool>("verifyChecksum", true)),
       useL1EventID_(pset.getUntrackedParameter<bool>("useL1EventID", false)) {
-  itFileName_ = fileNames().begin();
+  itFileName_ = fileNames(0).begin();
   openFile(*itFileName_);
   produces<FEDRawDataCollection>();
 }
@@ -30,7 +30,7 @@ bool FRDStreamSource::setRunAndEventInfo(edm::EventID& id,
                                          edm::TimeValue_t& theTime,
                                          edm::EventAuxiliary::ExperimentType& eType) {
   if (fin_.peek() == EOF) {
-    if (++itFileName_ == fileNames().end()) {
+    if (++itFileName_ == fileNames(0).end()) {
       fin_.close();
       return false;
     }

FWCore/Catalog/interface/FileLocator.h

@@ -12,7 +12,7 @@ namespace edm {
 
   class FileLocator {
   public:
-    explicit FileLocator(std::string const& catUrl, bool fallback);
+    explicit FileLocator(std::string const& catUrl, unsigned iCatalog = 0);
     ~FileLocator();
 
     std::string pfn(std::string const& ilfn) const;
@@ -34,7 +34,7 @@ namespace edm {
     typedef std::vector<Rule> Rules;
     typedef std::map<std::string, Rules> ProtocolRules;
 
-    void init(std::string const& catUrl, bool fallback);
+    void init(std::string const& catUrl, unsigned iCatalog);
 
     void parseRule(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules);
 

FWCore/Catalog/interface/InputFileCatalog.h

@@ -2,7 +2,9 @@
 #define FWCore_Catalog_InputFileCatalog_h
 //////////////////////////////////////////////////////////////////////
 //
-// Class InputFileCatalog. Services to manage InputFile catalog
+// Class InputFileCatalog. Services to manage InputFile catalog.
+// Physical file names, pfns_ of FileCatalogItem, are constructed from multiple data catalogs in site-local-config.xml. Each member of pfns_ corresponds to a data catalog.
+// Note that fileNames(unsigned iCatalog) of InputFileCatalog return physical file names of all input files corresponding to a data catalog (for example, a job has 10 input files provided as a PoolSource, the fileNames(unsigned iCatalog) will return PFNs of these 10 files constructed from a data catalog)
 //
 //////////////////////////////////////////////////////////////////////
 
@@ -15,47 +17,40 @@
 namespace edm {
   class FileCatalogItem {
   public:
-    FileCatalogItem() : pfn_(), lfn_(), fallbackPfn_() {}
-    FileCatalogItem(std::string const& pfn, std::string const& lfn, std::string const& fallbackPfn)
-        : pfn_(pfn), lfn_(lfn), fallbackPfn_(fallbackPfn) {}
-    std::string const& fileName() const { return pfn_; }
+    FileCatalogItem(std::vector<std::string> const& pfns, std::string const& lfn) : pfns_(pfns), lfn_(lfn) {}
+
+    std::string const& fileName(unsigned iCatalog) const { return pfns_[iCatalog]; }
     std::string const& logicalFileName() const { return lfn_; }
-    std::string const& fallbackFileName() const { return fallbackPfn_; }
+
+    std::vector<std::string> const& fileNames() const { return pfns_; }
 
   private:
-    std::string pfn_;
+    std::vector<std::string> pfns_;
     std::string lfn_;
-    std::string fallbackPfn_;
   };
 
   class InputFileCatalog {
   public:
     InputFileCatalog(std::vector<std::string> const& fileNames,
                      std::string const& override,
                      bool useLFNasPFNifLFNnotFound = false);
-    InputFileCatalog(std::vector<std::string> const& fileNames,
-                     std::string const& override,
-                     std::string const& overrideFallback,
-                     bool useLFNasPFNifLFNnotFound = false);
+
     ~InputFileCatalog();
     std::vector<FileCatalogItem> const& fileCatalogItems() const { return fileCatalogItems_; }
     std::vector<std::string> const& logicalFileNames() const { return logicalFileNames_; }
-    std::vector<std::string> const& fileNames() const { return fileNames_; }
-    std::vector<std::string> const& fallbackFileNames() const { return fallbackFileNames_; }
+    std::vector<std::string> fileNames(unsigned iCatalog) const;
     bool empty() const { return fileCatalogItems_.empty(); }
     static bool isPhysical(std::string const& name) { return (name.empty() || name.find(':') != std::string::npos); }
 
   private:
-    void init(std::string const& override, std::string const& overrideFallback, bool useLFNasPFNifLFNnotFound);
-    void findFile(std::string& pfn, std::string& fallbackPfn, std::string const& lfn, bool useLFNasPFNifLFNnotFound);
+    void init(std::string const& override, bool useLFNasPFNifLFNnotFound);
+    void findFile(std::string const& lfn, std::vector<std::string>& pfns, bool useLFNasPFNifLFNnotFound);
     std::vector<std::string> logicalFileNames_;
     std::vector<std::string> fileNames_;
-    std::vector<std::string> fallbackFileNames_;
     std::vector<FileCatalogItem> fileCatalogItems_;
-    edm::propagate_const<std::unique_ptr<FileLocator>> fileLocator_;
     edm::propagate_const<std::unique_ptr<FileLocator>> overrideFileLocator_;
-    edm::propagate_const<std::unique_ptr<FileLocator>> fallbackFileLocator_;
-    edm::propagate_const<std::unique_ptr<FileLocator>> overrideFallbackFileLocator_;
+
+    std::vector<edm::propagate_const<std::unique_ptr<FileLocator>>> fileLocators_;
   };
 }  // namespace edm
 

FWCore/Catalog/interface/SiteLocalConfig.h

@@ -1,5 +1,10 @@
 #ifndef FWCore_Catalog_SiteLocalConfig_h
 #define FWCore_Catalog_SiteLocalConfig_h
+////////////////////////////////////////////////////////////
+//
+// Abstract class. dataCatalogs() returns multiple data catalogs from site-local-config.xml. It is overridden in derived classes.
+//
+////////////////////////////////////////////////////////////
 
 // INCLUDES
 #include <set>
@@ -25,8 +30,7 @@ namespace edm {
     SiteLocalConfig() {}
     virtual ~SiteLocalConfig() {}
 
-    virtual std::string const dataCatalog(void) const = 0;
-    virtual std::string const fallbackDataCatalog(void) const = 0;
+    virtual std::vector<std::string> const& dataCatalogs(void) const = 0;
     virtual std::string const lookupCalibConnect(std::string const& input) const = 0;
     virtual std::string const rfioType(void) const = 0;
 

FWCore/Catalog/src/FileLocator.cc

@@ -42,9 +42,8 @@ namespace {
 }  // namespace
 
 namespace edm {
-
-  FileLocator::FileLocator(std::string const& catUrl, bool fallback) : m_destination("any") {
-    init(catUrl, fallback);
+  FileLocator::FileLocator(std::string const& catUrl, unsigned iCatalog) : m_destination("any") {
+    init(catUrl, iCatalog);
 
     // std::cout << m_protocols.size() << " protocols" << std::endl;
     // std::cout << m_directRules[m_protocols[0]].size() << " rules" << std::endl;
@@ -90,19 +89,18 @@ namespace edm {
     rules[protocol].emplace_back(std::move(rule));
   }
 
-  void FileLocator::init(std::string const& catUrl, bool fallback) {
+  void FileLocator::init(std::string const& catUrl, unsigned iCatalog) {
     std::string m_url = catUrl;
 
     if (m_url.empty()) {
       Service<SiteLocalConfig> localconfservice;
       if (!localconfservice.isAvailable())
         throw cms::Exception("TrivialFileCatalog", "edm::SiteLocalConfigService is not available");
-
-      m_url = (fallback ? localconfservice->fallbackDataCatalog() : localconfservice->dataCatalog());
+      if (iCatalog >= localconfservice->dataCatalogs().size())
+        throw cms::Exception("TrivialFileCatalog", "edm::FileLocator: Request nonexistence data catalog");
+      m_url = localconfservice->dataCatalogs()[iCatalog];
     }
 
-    // std::cout << "Connecting to the catalog " << m_url << std::endl;
-
     if (m_url.find("file:") == std::string::npos) {
       throw cms::Exception("TrivialFileCatalog",
                            "TrivialFileCatalog::connect: Malformed url for file catalog configuration");
@@ -164,7 +162,6 @@ namespace edm {
       throw cms::Exception("TrivialFileCatalog")
           << "tinyxml file load failed with error : " << doc.ErrorStr() << std::endl;
     }
-
     /* trivialFileCatalog matches the following xml schema
 	 FIXME: write a proper DTD
 	 <storage-mapping>
@@ -175,8 +172,7 @@ namespace edm {
 	 path-match="lfn/guid match regular expression"
 	 result="$1"/>
 	 </storage-mapping>
-      */
-
+    */
     auto rootElement = doc.RootElement();
     /*first of all do the lfn-to-pfn bit*/
     for (auto el = rootElement->FirstChildElement("lfn-to-pfn"); el != nullptr;