From 63ca0fdb150cbe12fcf0fe04c44036e5023bba18 Mon Sep 17 00:00:00 2001 From: Duong Date: Thu, 2 Mar 2023 13:29:57 -0600 Subject: [PATCH] handle invalid entries in --- FWCore/Catalog/src/FileLocator.cc | 9 +++--- FWCore/Catalog/src/InputFileCatalog.cc | 44 ++++++++++++++++++-------- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/FWCore/Catalog/src/FileLocator.cc b/FWCore/Catalog/src/FileLocator.cc index 522850d4e7182..b22acebe754eb 100644 --- a/FWCore/Catalog/src/FileLocator.cc +++ b/FWCore/Catalog/src/FileLocator.cc @@ -268,8 +268,8 @@ namespace edm { //let enforce that site-local-config.xml and storage.json contains valid catalogs in , in which site defined in site-local-config.xml should be found in storage.json if (found_site == json.end()) { cms::Exception ex("FileCatalog"); - ex << "Can not find site and volume " << aCatalog.site << ", " << aCatalog.volume - << " in storage.json. Check site-local-config.xml and storage.json"; + ex << "Can not find storage site \"" << aCatalog.storageSite << "\" and volume \"" << aCatalog.volume + << "\" in storage.json. Check site-local-config.xml and storage.json"; ex.addContext("edm::FileLocator:init()"); throw ex; } @@ -283,8 +283,9 @@ namespace edm { //let enforce that site-local-config.xml and storage.json contains valid catalogs, in which protocol defined in site-local-config.xml should be found in storage.json if (found_protocol == protocols.end()) { cms::Exception ex("FileCatalog"); - ex << "Can not find protocol " << aCatalog.protocol - << " in storage.json. Check site-local-config.xml and storage.json"; + ex << "Can not find protocol \"" << aCatalog.protocol << "\" for the storage site \"" << aCatalog.storageSite + << "\" and volume \"" << aCatalog.volume + << "\" in storage.json. Check site-local-config.xml and storage.json"; ex.addContext("edm::FileLocator:init()"); throw ex; } diff --git a/FWCore/Catalog/src/InputFileCatalog.cc b/FWCore/Catalog/src/InputFileCatalog.cc index 26b51d7504240..c3eddb052f96a 100644 --- a/FWCore/Catalog/src/InputFileCatalog.cc +++ b/FWCore/Catalog/src/InputFileCatalog.cc @@ -9,6 +9,7 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" #include @@ -56,8 +57,11 @@ namespace edm { throw ex; } - edm::CatalogAttributes inputOverride_struct( - tmps[0], tmps[1], tmps[2], tmps[3], tmps[4]); //site, subSite,storageSite,volume,protocol + edm::CatalogAttributes inputOverride_struct(tmps[0], //current-site + tmps[1], //current-subSite + tmps[2], //desired-data-access-site + tmps[3], //desired-data-access-volume + tmps[4]); //desired-data-access-protocol overrideFileLocator_ = std::make_unique(inputOverride_struct); // propagate_const has no reset() function @@ -76,30 +80,42 @@ namespace edm { std::vector const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs(); if (!fileLocators_trivalCatalog_.empty()) fileLocators_trivalCatalog_.clear(); - - //require the first file locator to success so obvious mistakes in data catalogs, typos for example, can be catched early. Note that tmp_dataCatalogs is not empty at this point. The protection is done inside the trivialDataCatalogs() of SiteLocalConfigService - fileLocators_trivalCatalog_.push_back(std::make_unique(tmp_dataCatalogs.front())); - - for (auto it = tmp_dataCatalogs.begin() + 1; it != tmp_dataCatalogs.end(); ++it) { + //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init). + for (const auto& catalog : tmp_dataCatalogs) { try { - fileLocators_trivalCatalog_.push_back(std::make_unique(*it)); + fileLocators_trivalCatalog_.push_back(std::make_unique(catalog)); } catch (cms::Exception const& e) { - continue; + edm::LogWarning("InputFileCatalog") + << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what() + << "Skip this catalog"; } } + if (fileLocators_trivalCatalog_.empty()) { + cms::Exception ex("FileCatalog"); + ex << "Unable to construct any file locator in InputFileCatalog::init"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } } else if (catType == edm::CatalogType::RucioCatalog) { std::vector const& tmp_dataCatalogs = localconfservice->dataCatalogs(); if (!fileLocators_.empty()) fileLocators_.clear(); - //require the first file locator to success so obvious mistakes in data catalogs, typos for example, can be catched early. Note that tmp_dataCatalogs is not empty at this point. The protection is done inside the dataCatalogs() of SiteLocalConfigService - fileLocators_.push_back(std::make_unique(tmp_dataCatalogs.front())); - for (auto it = tmp_dataCatalogs.begin() + 1; it != tmp_dataCatalogs.end(); ++it) { + //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init). + for (const auto& catalog : tmp_dataCatalogs) { try { - fileLocators_.push_back(std::make_unique(*it)); + fileLocators_.push_back(std::make_unique(catalog)); } catch (cms::Exception const& e) { - continue; + edm::LogWarning("InputFileCatalog") + << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what() + << "Skip this catalog"; } } + if (fileLocators_.empty()) { + cms::Exception ex("FileCatalog"); + ex << "Unable to construct any file locator in InputFileCatalog::init"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } } else { cms::Exception ex("FileCatalog"); ex << "Undefined catalog type";