diff --git a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py index 6f32f6e814c90..32ebb34d3710f 100755 --- a/Configuration/PyReleaseValidation/scripts/runTheMatrix.py +++ b/Configuration/PyReleaseValidation/scripts/runTheMatrix.py @@ -399,6 +399,7 @@ def runSelected(opt): if os.path.exists(cmssw_base): os.environ["PATH"]=cmssw_base+":"+os.getenv("PATH") os.environ["CMS_PATH"]="/cvmfs/cms-ib.cern.ch" + os.environ["SITECONFIG_PATH"]="/cvmfs/cms-ib.cern.ch/SITECONF/local" os.environ["CMSSW_USE_IBEOS"]="true" print(">> WARNING: You are using SITECONF from /cvmfs/cms-ib.cern.ch") break diff --git a/FWCore/Catalog/interface/FileLocator.h b/FWCore/Catalog/interface/FileLocator.h index 8e13a0f15fb69..1c4765cfbf3bd 100644 --- a/FWCore/Catalog/interface/FileLocator.h +++ b/FWCore/Catalog/interface/FileLocator.h @@ -1,22 +1,29 @@ #ifndef FWCore_Catalog_FileLocator_h #define FWCore_Catalog_FileLocator_h +#include "FWCore/Catalog/interface/SiteLocalConfig.h" #include #include #include #include #include #include "tinyxml2.h" +#include namespace edm { class FileLocator { public: + explicit FileLocator( + edm::CatalogAttributes const& catAttr, + unsigned iCatalog = 0, + //storageDescriptionPath is used to override path provided by SiteLocalConfig. This is used in FileLocator_t.cpp tests + std::string const& storageDescriptionPath = std::string()); explicit FileLocator(std::string const& catUrl, unsigned iCatalog = 0); + ~FileLocator(); - std::string pfn(std::string const& ilfn) const; - std::string lfn(std::string const& ipfn) const; + std::string pfn(std::string const& ilfn, edm::CatalogType catType) const; private: /** For the time being the only allowed configuration item is a @@ -34,9 +41,15 @@ namespace edm { typedef std::vector Rules; typedef std::map ProtocolRules; - void init(std::string const& catUrl, unsigned iCatalog); - - void parseRule(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules); + void init_trivialCatalog(std::string const& catUrl, unsigned iCatalog); + void parseRuleTrivialCatalog(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules); + //using data-access + void init(edm::CatalogAttributes const& input_dataCatalog, + unsigned iCatalog, + std::string const& storageDescriptionPath); + void parseRule(boost::property_tree::ptree::value_type const& storageRule, + std::string const& protocol, + ProtocolRules& rules); std::string applyRules(ProtocolRules const& protocolRules, std::string const& protocol, @@ -47,14 +60,17 @@ namespace edm { std::string convert(std::string const& input, ProtocolRules const& rules, bool direct) const; /** Direct rules are used to do the mapping from LFN to PFN.*/ - ProtocolRules m_directRules; + ProtocolRules m_directRules_trivialCatalog; /** Inverse rules are used to do the mapping from PFN to LFN*/ ProtocolRules m_inverseRules; + /** Direct rules are used to do the mapping from LFN to PFN taken from storage.json*/ + ProtocolRules m_directRules; std::string m_fileType; std::string m_filename; std::vector m_protocols; std::string m_destination; + std::string m_prefix; }; } // namespace edm diff --git a/FWCore/Catalog/interface/InputFileCatalog.h b/FWCore/Catalog/interface/InputFileCatalog.h index d206ace5e5ab5..638cad95e7589 100644 --- a/FWCore/Catalog/interface/InputFileCatalog.h +++ b/FWCore/Catalog/interface/InputFileCatalog.h @@ -5,6 +5,8 @@ // Class InputFileCatalog. Services to manage InputFile catalog. // Physical file names, pfns_ of FileCatalogItem, are constructed from multiple data catalogs in site-local-config.xml. Each member of pfns_ corresponds to a data catalog. // Note that fileNames(unsigned iCatalog) of InputFileCatalog return physical file names of all input files corresponding to a data catalog (for example, a job has 10 input files provided as a PoolSource, the fileNames(unsigned iCatalog) will return PFNs of these 10 files constructed from a data catalog) +// Set catType=TrivialCatalog: use trivial data catalogs from +// Set catType=RucioCatalog: use data catalogs from and storage.json // ////////////////////////////////////////////////////////////////////// @@ -33,7 +35,10 @@ namespace edm { public: InputFileCatalog(std::vector const& fileNames, std::string const& override, - bool useLFNasPFNifLFNnotFound = false); + bool useLFNasPFNifLFNnotFound = false, + //switching between two catalog types + //edm::CatalogType catType = edm::CatalogType::TrivialCatalog); + edm::CatalogType catType = edm::CatalogType::RucioCatalog); ~InputFileCatalog(); std::vector const& fileCatalogItems() const { return fileCatalogItems_; } @@ -43,13 +48,17 @@ namespace edm { static bool isPhysical(std::string const& name) { return (name.empty() || name.find(':') != std::string::npos); } private: - void init(std::string const& override, bool useLFNasPFNifLFNnotFound); - void findFile(std::string const& lfn, std::vector& pfns, bool useLFNasPFNifLFNnotFound); + void init(std::string const& override, bool useLFNasPFNifLFNnotFound, edm::CatalogType catType); + void findFile(std::string const& lfn, + std::vector& pfns, + bool useLFNasPFNifLFNnotFound, + edm::CatalogType catType); std::vector logicalFileNames_; std::vector fileNames_; std::vector fileCatalogItems_; edm::propagate_const> overrideFileLocator_; + std::vector>> fileLocators_trivalCatalog_; std::vector>> fileLocators_; }; } // namespace edm diff --git a/FWCore/Catalog/interface/SiteLocalConfig.h b/FWCore/Catalog/interface/SiteLocalConfig.h index 602bfbc9df052..d8b4ec8e57c0d 100644 --- a/FWCore/Catalog/interface/SiteLocalConfig.h +++ b/FWCore/Catalog/interface/SiteLocalConfig.h @@ -11,13 +11,40 @@ #include #include #include - +#include // PUBLIC DEFINES // PUBLIC CONSTANTS // PUBLIC TYPES namespace edm { class ParameterSet; class ActivityRegistry; + + //attributes of a data catalog (Rucio format) defined in block of site-local-config.xml. See further description in SiteLocalConfigService.cc + struct CatalogAttributes { + CatalogAttributes() = default; + CatalogAttributes(std::string input_site, + std::string input_subSite, + std::string input_storageSite, + std::string input_volume, + std::string input_protocol) + : site(std::move(input_site)), + subSite(std::move(input_subSite)), + storageSite(std::move(input_storageSite)), + volume(std::move(input_volume)), + protocol(std::move(input_protocol)) {} + bool operator==(const CatalogAttributes& aCatalog) const { + return site == aCatalog.site && subSite == aCatalog.subSite && storageSite == aCatalog.storageSite && + volume == aCatalog.volume && protocol == aCatalog.protocol; + } + bool empty() const { return site.empty() && storageSite.empty() && volume.empty() && protocol.empty(); } + std::string site; + std::string subSite; + std::string storageSite; //site where storage description is used + std::string volume; + std::string protocol; + }; + + enum class CatalogType { TrivialCatalog, RucioCatalog }; } // namespace edm // PUBLIC VARIABLES @@ -30,7 +57,9 @@ namespace edm { SiteLocalConfig() {} virtual ~SiteLocalConfig() {} - virtual std::vector const& dataCatalogs(void) const = 0; + virtual std::vector const& trivialDataCatalogs() const = 0; + virtual std::vector const& dataCatalogs() const = 0; + virtual std::filesystem::path const storageDescriptionPath(const edm::CatalogAttributes& aDataCatalog) const = 0; virtual std::string const lookupCalibConnect(std::string const& input) const = 0; virtual std::string const rfioType(void) const = 0; @@ -47,6 +76,7 @@ namespace edm { virtual struct addrinfo const* statisticsDestination() const = 0; virtual std::set const* statisticsInfo() const = 0; virtual std::string const& siteName(void) const = 0; + virtual std::string const& subSiteName(void) const = 0; virtual bool useLocalConnectString() const = 0; virtual std::string const& localConnectPrefix() const = 0; virtual std::string const& localConnectSuffix() const = 0; diff --git a/FWCore/Catalog/src/FileLocator.cc b/FWCore/Catalog/src/FileLocator.cc index 31c2e63c848bb..b22acebe754eb 100644 --- a/FWCore/Catalog/src/FileLocator.cc +++ b/FWCore/Catalog/src/FileLocator.cc @@ -1,36 +1,36 @@ #include "FWCore/Catalog/interface/FileLocator.h" -#include "FWCore/Catalog/interface/SiteLocalConfig.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include #include +#include +#include #include #include #include -#include #include +namespace pt = boost::property_tree; + namespace { std::string replaceWithRegexp(std::smatch const& matches, std::string const& outputFormat) { std::string result = outputFormat; std::stringstream str; - // std::cerr << "Output format: "<< outputFormat << std::endl; for (size_t i = 1; i < matches.size(); ++i) { str.str(""); str << "$" << i; - // std::cerr << "Current match: " << matches[i] << std::endl; std::string const matchedString(matches[i].first, matches[i].second); if (!matchedString.empty()) boost::algorithm::replace_all(result, str.str(), matchedString); } - // std::cerr << "Final string: " << result << std::endl; return result; } constexpr char const* const kEmptyString = ""; + constexpr char const* const kLFNPrefix = "/store/"; const char* safe(const char* iCheck) { if (iCheck == nullptr) { @@ -41,32 +41,52 @@ namespace { } // namespace +namespace pt = boost::property_tree; + namespace edm { + FileLocator::FileLocator(std::string const& catUrl, unsigned iCatalog) : m_destination("any") { - init(catUrl, iCatalog); + init_trivialCatalog(catUrl, iCatalog); + } - // std::cout << m_protocols.size() << " protocols" << std::endl; - // std::cout << m_directRules[m_protocols[0]].size() << " rules" << std::endl; + FileLocator::FileLocator(edm::CatalogAttributes const& catAttr, + unsigned iCatalog, + std::string const& storageDescriptionPath) + : m_destination("any") { + init(catAttr, iCatalog, storageDescriptionPath); } FileLocator::~FileLocator() {} - std::string FileLocator::pfn(std::string const& ilfn) const { return convert(ilfn, m_directRules, true); } - - std::string FileLocator::lfn(std::string const& ipfn) const { return convert(ipfn, m_inverseRules, false); } + std::string FileLocator::pfn(std::string const& ilfn, edm::CatalogType catType) const { + if (catType == edm::CatalogType::TrivialCatalog) + return convert(ilfn, m_directRules_trivialCatalog, true); + return convert(ilfn, m_directRules, true); + } std::string FileLocator::convert(std::string const& input, ProtocolRules const& rules, bool direct) const { std::string out = ""; - + //check if input is an authentic LFN + if (input.compare(0, 7, kLFNPrefix) != 0) + return out; + //use prefix in the protocol + if (!m_prefix.empty()) { + out = m_prefix + "/" + input; + if (input[0] == '/') + out = m_prefix + input; + return out; + } + //no prefix in the protocol, use rule for (size_t pi = 0, pe = m_protocols.size(); pi != pe; ++pi) { out = applyRules(rules, m_protocols[pi], m_destination, direct, input); - if (!out.empty()) + if (!out.empty()) { return out; + } } return out; } - void FileLocator::parseRule(tinyxml2::XMLElement* ruleElement, ProtocolRules& rules) { + void FileLocator::parseRuleTrivialCatalog(tinyxml2::XMLElement* ruleElement, ProtocolRules& rules) { if (!ruleElement) { throw cms::Exception("TrivialFileCatalog", std::string("TrivialFileCatalog::connect: Malformed trivial catalog")); } @@ -89,27 +109,42 @@ namespace edm { rules[protocol].emplace_back(std::move(rule)); } - void FileLocator::init(std::string const& catUrl, unsigned iCatalog) { - std::string m_url = catUrl; + void FileLocator::parseRule(pt::ptree::value_type const& storageRule, + std::string const& protocol, + ProtocolRules& rules) { + if (storageRule.second.empty()) { + throw cms::Exception("RucioFileCatalog", "edm::FileLocator::parseRule Malformed storage rule"); + } + auto const pathMatchRegexp = storageRule.second.get("lfn"); + auto const result = storageRule.second.get("pfn"); + Rule rule; + rule.pathMatch.assign(pathMatchRegexp); + rule.destinationMatch.assign(".*"); + rule.result = result; + rule.chain = ""; + rules[protocol].emplace_back(std::move(rule)); + } - if (m_url.empty()) { + void FileLocator::init_trivialCatalog(std::string const& catUrl, unsigned iCatalog) { + std::string url = catUrl; + if (url.empty()) { Service localconfservice; if (!localconfservice.isAvailable()) throw cms::Exception("TrivialFileCatalog", "edm::SiteLocalConfigService is not available"); - if (iCatalog >= localconfservice->dataCatalogs().size()) + if (iCatalog >= localconfservice->trivialDataCatalogs().size()) throw cms::Exception("TrivialFileCatalog", "edm::FileLocator: Request nonexistence data catalog"); - m_url = localconfservice->dataCatalogs()[iCatalog]; + url = localconfservice->trivialDataCatalogs()[iCatalog]; } - if (m_url.find("file:") == std::string::npos) { + if (url.find("file:") == std::string::npos) { throw cms::Exception("TrivialFileCatalog", "TrivialFileCatalog::connect: Malformed url for file catalog configuration"); } - m_url = m_url.erase(0, m_url.find(':') + 1); + url = url.erase(0, url.find(':') + 1); std::vector tokens; - boost::algorithm::split(tokens, m_url, boost::is_any_of(std::string("?"))); + boost::algorithm::split(tokens, url, boost::is_any_of(std::string("?"))); m_filename = tokens[0]; if (tokens.size() == 2) { @@ -146,9 +181,6 @@ namespace edm { std::ifstream configFile; configFile.open(m_filename.c_str()); - // - // std::cout << "Using catalog configuration " << m_filename << std::endl; - if (!configFile.good() || !configFile.is_open()) { throw cms::Exception("TrivialFileCatalog", "TrivialFileCatalog::connect: Unable to open trivial file catalog " + m_filename); @@ -177,13 +209,97 @@ namespace edm { /*first of all do the lfn-to-pfn bit*/ for (auto el = rootElement->FirstChildElement("lfn-to-pfn"); el != nullptr; el = el->NextSiblingElement("lfn-to-pfn")) { - parseRule(el, m_directRules); + parseRuleTrivialCatalog(el, m_directRules_trivialCatalog); } /*Then we handle the pfn-to-lfn bit*/ for (auto el = rootElement->FirstChildElement("pfn-to-lfn"); el != nullptr; el = el->NextSiblingElement("pfn-to-lfn")) { - parseRule(el, m_inverseRules); + parseRuleTrivialCatalog(el, m_inverseRules); + } + } + + void FileLocator::init(edm::CatalogAttributes const& input_dataCatalog, + unsigned iCatalog, + std::string const& storageDescriptionPath) { + Service localconfservice; + edm::CatalogAttributes aCatalog = input_dataCatalog; + if (input_dataCatalog.empty()) { + if (!localconfservice.isAvailable()) { + cms::Exception ex("FileCatalog"); + ex << "edm::SiteLocalConfigService is not available"; + ex.addContext("Calling edm::FileLocator::init()"); + throw ex; + } + if (iCatalog >= localconfservice->dataCatalogs().size()) { + cms::Exception ex("FileCatalog"); + ex << "Request nonexistence data catalog"; + ex.addContext("Calling edm::FileLocator::init()"); + throw ex; + } + aCatalog = localconfservice->dataCatalogs()[iCatalog]; + } + + std::filesystem::path filename_storage = localconfservice->storageDescriptionPath(aCatalog); + + //use path to storage description from input parameter + if (!storageDescriptionPath.empty()) + filename_storage = storageDescriptionPath; + + //now read json + pt::ptree json; + try { + boost::property_tree::read_json(filename_storage.string(), json); + } catch (std::exception& e) { + cms::Exception ex("FileCatalog"); + ex << "Can not open storage.json (" << filename_storage.string() + << "). Check SITECONFIG_PATH and site-local-config.xml "; + ex.addContext("edm::FileLocator:init()"); + throw ex; + } + auto found_site = std::find_if(json.begin(), json.end(), [&](pt::ptree::value_type const& site) { + //get site name + std::string siteName = site.second.get("site", kEmptyString); + //get volume name + std::string volName = site.second.get("volume", kEmptyString); + return aCatalog.storageSite == siteName && aCatalog.volume == volName; + }); + + //let enforce that site-local-config.xml and storage.json contains valid catalogs in , in which site defined in site-local-config.xml should be found in storage.json + if (found_site == json.end()) { + cms::Exception ex("FileCatalog"); + ex << "Can not find storage site \"" << aCatalog.storageSite << "\" and volume \"" << aCatalog.volume + << "\" in storage.json. Check site-local-config.xml and storage.json"; + ex.addContext("edm::FileLocator:init()"); + throw ex; + } + + const pt::ptree& protocols = found_site->second.find("protocols")->second; + auto found_protocol = std::find_if(protocols.begin(), protocols.end(), [&](pt::ptree::value_type const& protocol) { + std::string protName = protocol.second.get("protocol", kEmptyString); + return aCatalog.protocol == protName; + }); + + //let enforce that site-local-config.xml and storage.json contains valid catalogs, in which protocol defined in site-local-config.xml should be found in storage.json + if (found_protocol == protocols.end()) { + cms::Exception ex("FileCatalog"); + ex << "Can not find protocol \"" << aCatalog.protocol << "\" for the storage site \"" << aCatalog.storageSite + << "\" and volume \"" << aCatalog.volume + << "\" in storage.json. Check site-local-config.xml and storage.json"; + ex.addContext("edm::FileLocator:init()"); + throw ex; + } + + std::string protName = found_protocol->second.get("protocol", kEmptyString); + m_protocols.push_back(protName); + m_prefix = found_protocol->second.get("prefix", kEmptyString); + if (m_prefix == kEmptyString) { + //get rules + const pt::ptree& rules = found_protocol->second.find("rules")->second; + //loop over rules + for (pt::ptree::value_type const& storageRule : rules) { + parseRule(storageRule, protName, m_directRules); + } } } @@ -192,8 +308,6 @@ namespace edm { std::string const& destination, bool direct, std::string name) const { - // std::cerr << "Calling apply rules with protocol: " << protocol << "\n destination: " << destination << "\n " << " on name " << name << std::endl; - ProtocolRules::const_iterator const rulesIterator = protocolRules.find(protocol); if (rulesIterator == protocolRules.end()) { return ""; diff --git a/FWCore/Catalog/src/InputFileCatalog.cc b/FWCore/Catalog/src/InputFileCatalog.cc index 51a8082a589a0..c3eddb052f96a 100644 --- a/FWCore/Catalog/src/InputFileCatalog.cc +++ b/FWCore/Catalog/src/InputFileCatalog.cc @@ -9,16 +9,20 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" #include +#include + namespace edm { InputFileCatalog::InputFileCatalog(std::vector const& fileNames, std::string const& override, - bool useLFNasPFNifLFNnotFound) + bool useLFNasPFNifLFNnotFound, + edm::CatalogType catType) : logicalFileNames_(fileNames), fileNames_(fileNames), fileCatalogItems_(), overrideFileLocator_() { - init(override, useLFNasPFNifLFNnotFound); + init(override, useLFNasPFNifLFNnotFound, catType); } InputFileCatalog::~InputFileCatalog() {} @@ -32,31 +36,91 @@ namespace edm { return tmp; } - void InputFileCatalog::init(std::string const& inputOverride, bool useLFNasPFNifLFNnotFound) { + void InputFileCatalog::init(std::string const& inputOverride, + bool useLFNasPFNifLFNnotFound, + edm::CatalogType catType) { typedef std::vector::iterator iter; if (!overrideFileLocator_ && !inputOverride.empty()) { - overrideFileLocator_ = - std::make_unique(inputOverride); // propagate_const has no reset() function - } + if (catType == edm::CatalogType::TrivialCatalog) { + overrideFileLocator_ = + std::make_unique(inputOverride); // propagate_const has no reset() function + } else if (catType == edm::CatalogType::RucioCatalog) { + //now make a struct from input string + std::vector tmps; + boost::algorithm::split(tmps, inputOverride, boost::is_any_of(std::string(","))); + if (tmps.size() != 5) { + cms::Exception ex("FileCatalog"); + ex << "Trying to override input file catalog but invalid input override string " << inputOverride + << " (Should be site,subSite,storageSite,volume,protocol)"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } - Service localconfservice; - if (!localconfservice.isAvailable()) - throw cms::Exception("TrivialFileCatalog", "edm::SiteLocalConfigService is not available"); + edm::CatalogAttributes inputOverride_struct(tmps[0], //current-site + tmps[1], //current-subSite + tmps[2], //desired-data-access-site + tmps[3], //desired-data-access-volume + tmps[4]); //desired-data-access-protocol - std::vector const& tmp_dataCatalogs = localconfservice->dataCatalogs(); - if (!fileLocators_.empty()) - fileLocators_.clear(); + overrideFileLocator_ = + std::make_unique(inputOverride_struct); // propagate_const has no reset() function + } + } - //require the first file locator to success so obvious mistakes in data catalogs, typos for example, can be catched early. Note that tmp_dataCatalogs is not empty at this point. The protection is done inside the dataCatalogs() above - fileLocators_.push_back(std::make_unique(tmp_dataCatalogs.front())); + Service localconfservice; + if (!localconfservice.isAvailable()) { + cms::Exception ex("FileCatalog"); + ex << "edm::SiteLocalConfigService is not available"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } - for (auto it = tmp_dataCatalogs.begin() + 1; it != tmp_dataCatalogs.end(); ++it) { - try { - fileLocators_.push_back(std::make_unique(*it)); - } catch (cms::Exception const& e) { - continue; + if (catType == edm::CatalogType::TrivialCatalog) { + std::vector const& tmp_dataCatalogs = localconfservice->trivialDataCatalogs(); + if (!fileLocators_trivalCatalog_.empty()) + fileLocators_trivalCatalog_.clear(); + //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init). + for (const auto& catalog : tmp_dataCatalogs) { + try { + fileLocators_trivalCatalog_.push_back(std::make_unique(catalog)); + } catch (cms::Exception const& e) { + edm::LogWarning("InputFileCatalog") + << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what() + << "Skip this catalog"; + } } + if (fileLocators_trivalCatalog_.empty()) { + cms::Exception ex("FileCatalog"); + ex << "Unable to construct any file locator in InputFileCatalog::init"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } + } else if (catType == edm::CatalogType::RucioCatalog) { + std::vector const& tmp_dataCatalogs = localconfservice->dataCatalogs(); + if (!fileLocators_.empty()) + fileLocators_.clear(); + //Construct all file locators from data catalogs. If a data catalog is invalid (wrong protocol for example), it is skipped and no file locator is constructed (an exception is thrown out from FileLocator::init). + for (const auto& catalog : tmp_dataCatalogs) { + try { + fileLocators_.push_back(std::make_unique(catalog)); + } catch (cms::Exception const& e) { + edm::LogWarning("InputFileCatalog") + << "Caught an exception while constructing a file locator in InputFileCatalog::init: " << e.what() + << "Skip this catalog"; + } + } + if (fileLocators_.empty()) { + cms::Exception ex("FileCatalog"); + ex << "Unable to construct any file locator in InputFileCatalog::init"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; + } + } else { + cms::Exception ex("FileCatalog"); + ex << "Undefined catalog type"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; } for (iter it = fileNames_.begin(), lt = logicalFileNames_.begin(), itEnd = fileNames_.end(); it != itEnd; @@ -64,37 +128,58 @@ namespace edm { boost::trim(*it); std::vector pfns; if (it->empty()) { - throw Exception(errors::Configuration, "InputFileCatalog::InputFileCatalog()\n") - << "An empty string specified in the fileNames parameter for input source.\n"; + cms::Exception ex("FileCatalog"); + ex << "An empty string specified in the fileNames parameter for input source"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; } if (isPhysical(*it)) { if (it->back() == ':') { - throw Exception(errors::Configuration, "InputFileCatalog::InputFileCatalog()\n") - << "An empty physical file name specified in the fileNames parameter for input source.\n"; + cms::Exception ex("FileCatalog"); + ex << "An empty physical file name specified in the fileNames parameter for input source"; + ex.addContext("Calling edm::InputFileCatalog::init()"); + throw ex; } pfns.push_back(*it); // Clear the LFN. lt->clear(); } else { boost::trim(*lt); - findFile(*lt, pfns, useLFNasPFNifLFNnotFound); + findFile(*lt, pfns, useLFNasPFNifLFNnotFound, catType); } + fileCatalogItems_.push_back(FileCatalogItem(pfns, *lt)); } } void InputFileCatalog::findFile(std::string const& lfn, std::vector& pfns, - bool useLFNasPFNifLFNnotFound) { + bool useLFNasPFNifLFNnotFound, + edm::CatalogType catType) { if (overrideFileLocator_) { - pfns.push_back(overrideFileLocator_->pfn(lfn)); + pfns.push_back(overrideFileLocator_->pfn(lfn, catType)); } else { - for (auto const& locator : fileLocators_) { - std::string pfn = locator->pfn(lfn); - if (pfn.empty() && useLFNasPFNifLFNnotFound) - pfns.push_back(lfn); - else - pfns.push_back(pfn); + if (catType == edm::CatalogType::TrivialCatalog) { + for (auto const& locator : fileLocators_trivalCatalog_) { + std::string pfn = locator->pfn(lfn, edm::CatalogType::TrivialCatalog); + if (pfn.empty() && useLFNasPFNifLFNnotFound) + pfns.push_back(lfn); + else + pfns.push_back(pfn); + } + } else if (catType == edm::CatalogType::RucioCatalog) { + for (auto const& locator : fileLocators_) { + std::string pfn = locator->pfn(lfn, edm::CatalogType::RucioCatalog); + if (pfn.empty() && useLFNasPFNifLFNnotFound) + pfns.push_back(lfn); + else + pfns.push_back(pfn); + } + } else { + cms::Exception ex("FileCatalog"); + ex << "Undefined catalog type"; + ex.addContext("Calling edm::InputFileCatalog::findFile()"); + throw ex; } } diff --git a/FWCore/Catalog/test/FileLocator_t.cpp b/FWCore/Catalog/test/FileLocator_t.cpp index 28593833352c9..20314a222df37 100644 --- a/FWCore/Catalog/test/FileLocator_t.cpp +++ b/FWCore/Catalog/test/FileLocator_t.cpp @@ -12,8 +12,16 @@ namespace { class TestSiteLocalConfig : public edm::SiteLocalConfig { public: - TestSiteLocalConfig(std::vector catalogs) : m_catalogs(std::move(catalogs)) {} - std::vector const& dataCatalogs(void) const final { return m_catalogs; } + //constructor using trivial data catalogs + TestSiteLocalConfig(std::vector catalogs) : m_trivialCatalogs(std::move(catalogs)) {} + //constructor using Rucio data catalogs + TestSiteLocalConfig(std::vector catalogs) : m_catalogs(std::move(catalogs)) {} + std::vector const& trivialDataCatalogs() const final { return m_trivialCatalogs; } + std::vector const& dataCatalogs() const final { return m_catalogs; } + std::filesystem::path const storageDescriptionPath(const edm::CatalogAttributes& aDataCatalog) const final { + return std::filesystem::path(); + } + std::string const lookupCalibConnect(std::string const& input) const final { return std::string(); } std::string const rfioType(void) const final { return std::string(); } @@ -32,16 +40,73 @@ namespace { } std::set const* statisticsInfo() const final { return nullptr; } std::string const& siteName(void) const final { return m_emptyString; } + std::string const& subSiteName(void) const final { return m_emptyString; } bool useLocalConnectString() const final { return false; } std::string const& localConnectPrefix() const final { return m_emptyString; } std::string const& localConnectSuffix() const final { return m_emptyString; } private: - std::vector m_catalogs; + std::vector m_trivialCatalogs; + std::vector m_catalogs; + std::filesystem::path m_storageDescription_path; std::string m_emptyString; }; } // namespace +TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]") { + //catalog for testing "prefix" + edm::CatalogAttributes aCatalog; + aCatalog.site = "T1_US_FNAL"; + aCatalog.subSite = "T1_US_FNAL"; + aCatalog.storageSite = "T1_US_FNAL"; + aCatalog.volume = "American_Federation"; + aCatalog.protocol = "XRootD"; + std::vector tmp{aCatalog}; + //catalog for testing "rules" + aCatalog.site = "T1_US_FNAL"; + aCatalog.subSite = "T1_US_FNAL"; + aCatalog.storageSite = "T1_US_FNAL"; + aCatalog.volume = "FNAL_dCache_EOS"; + aCatalog.protocol = "XRootD"; + tmp.push_back(aCatalog); + + //create the services + edm::ServiceToken tempToken( + edm::ServiceRegistry::createContaining(std::unique_ptr(new TestSiteLocalConfig(tmp)))); + + std::string CMSSW_BASE(std::getenv("CMSSW_BASE")); + std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE")); + std::string file_name("/src/FWCore/Catalog/test/storage.json"); + std::string full_file_name = std::filesystem::exists((CMSSW_BASE + file_name).c_str()) + ? CMSSW_BASE + file_name + : CMSSW_RELEASE_BASE + file_name; + + SECTION("prefix") { + edm::ServiceRegistry::Operate operate(tempToken); + edm::CatalogAttributes tmp_cat; //empty catalog + edm::FileLocator fl(tmp_cat, 0, full_file_name); //use the first catalog provided by site local config + CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" == + fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog)); + } + SECTION("rule") { + edm::ServiceRegistry::Operate operate(tempToken); + edm::CatalogAttributes tmp_cat; //empty catalog + edm::FileLocator fl(tmp_cat, 1, full_file_name); //use the second catalog provided by site local config + const std::array lfn = {{"/bha/bho", + "bha", + "file:bha", + "file:/bha/bho", + "/castor/cern.ch/cms/bha/bho", + "rfio:/castor/cern.ch/cms/bha/bho", + "rfio:/bha/bho"}}; + CHECK("root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/group/bha/bho" == + fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog)); + for (auto file : lfn) { + CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog)); + } + } +} + TEST_CASE("FileLocator", "[filelocator]") { std::string CMSSW_BASE(std::getenv("CMSSW_BASE")); std::string CMSSW_RELEASE_BASE(std::getenv("CMSSW_RELEASE_BASE")); @@ -58,7 +123,7 @@ TEST_CASE("FileLocator", "[filelocator]") { //make the services available SECTION("standard") { edm::ServiceRegistry::Operate operate(tempToken); - edm::FileLocator fl("", 0); + edm::FileLocator fl(""); const std::array lfn = {{"/bha/bho", "bha", @@ -68,15 +133,9 @@ TEST_CASE("FileLocator", "[filelocator]") { "rfio:/castor/cern.ch/cms/bha/bho", "rfio:/bha/bho"}}; - CHECK("/storage/path/store/group/bha/bho" == fl.pfn("/store/group/bha/bho")); - for (auto file : lfn) { - CHECK("" == fl.pfn(file)); - } - - CHECK(fl.lfn("/storage/path/store/group/bha/bho") == "/store/group/bha/bho"); - CHECK(fl.lfn("/store/group/bha/bho") == ""); + CHECK("/storage/path/store/group/bha/bho" == fl.pfn("/store/group/bha/bho", edm::CatalogType::TrivialCatalog)); for (auto file : lfn) { - CHECK("" == fl.lfn(file)); + CHECK("" == fl.pfn(file, edm::CatalogType::TrivialCatalog)); } } @@ -103,13 +162,11 @@ TEST_CASE("FileLocator", "[filelocator]") { "/store/unmerged/relval/CMSSW_3_8_0_pre3/RelValZTT/GEN-SIM-DIGI-RAW-HLTDEBUG/START38_V2-v1/0666/" "80EC0BCD-D279-DF11-B1DB-0030487C90EE.root"; - CHECK("/FULL_PATH_TO_THE_FIRST_STEP_ROOT_FILE/80EC0BCD-D279-DF11-B1DB-0030487C90EE.root" == fl.pfn(overriden_file)); + CHECK("/FULL_PATH_TO_THE_FIRST_STEP_ROOT_FILE/80EC0BCD-D279-DF11-B1DB-0030487C90EE.root" == + fl.pfn(overriden_file, edm::CatalogType::TrivialCatalog)); for (auto f : lfn) { - CHECK("" == fl.pfn(f)); - } - for (auto f : lfn) { - CHECK("" == fl.lfn(f)); + CHECK("" == fl.pfn(f, edm::CatalogType::TrivialCatalog)); } } } diff --git a/FWCore/Catalog/test/storage.json b/FWCore/Catalog/test/storage.json new file mode 100644 index 0000000000000..fccc3f1f76e64 --- /dev/null +++ b/FWCore/Catalog/test/storage.json @@ -0,0 +1,34 @@ +[ + { "site": "T1_US_FNAL", + "volume": "FNAL_dCache_EOS", + "protocols": [ + { "protocol": "XRootD", + "access": "global-rw", + "comment": "xrootd write to dCache/EOS endpoint directly", + "rules": [ + { "lfn": "/+store/temp/user/(.*)", + "pfn": "root://cmseos.fnal.gov//eos/uscms/store/temp/user/$1" + }, + { "lfn": "/+store/(.*)", + "pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1" + } + ] + } + ], + "type": "DISK", + "rse": "T1_US_FNAL_Disk", + "fts": [ "https://cmsfts3.fnal.gov:8446", "https://fts3-cms.cern.ch:8446" ] + }, + { "site": "T1_US_FNAL", + "volume": "American_Federation", + "protocols": [ + { "protocol": "XRootD", + "access": "global-ro", + "prefix": "root://cmsxrootd.fnal.gov" + } + ], + "type": "DISK", + "rse": null, + "fts": [] + } +] diff --git a/FWCore/Services/src/SiteLocalConfigService.cc b/FWCore/Services/src/SiteLocalConfigService.cc index 3ee1add4a0551..9298bcca37b86 100644 --- a/FWCore/Services/src/SiteLocalConfigService.cc +++ b/FWCore/Services/src/SiteLocalConfigService.cc @@ -15,7 +15,6 @@ #include #include #include - //<<<<<< PRIVATE DEFINES >>>>>> //<<<<<< PRIVATE CONSTANTS >>>>>> //<<<<<< PRIVATE TYPES >>>>>> @@ -59,11 +58,11 @@ namespace { std::string defaultURL() { std::string returnValue; - const char *tmp = std::getenv("CMS_PATH"); + const char *tmp = std::getenv("SITECONFIG_PATH"); if (tmp) { returnValue = tmp; } - returnValue += "/SITECONF/local/JobConfig/site-local-config.xml"; + returnValue += "/JobConfig/site-local-config.xml"; return returnValue; } @@ -76,6 +75,7 @@ namespace edm { SiteLocalConfigService::SiteLocalConfigService(ParameterSet const &pset) : m_url(pset.getUntrackedParameter("siteLocalConfigFileUrl", defaultURL())), + m_trivialDataCatalogs(), m_dataCatalogs(), m_frontierConnect(), m_rfioType("castor"), @@ -102,7 +102,8 @@ namespace edm { m_statisticsDestination(), m_statisticsAddrInfo(nullptr), m_statisticsInfoAvail(false), - m_siteName() { + m_siteName(), + m_subSiteName() { this->parse(m_url); //apply overrides @@ -150,22 +151,72 @@ namespace edm { } } - std::vector const &SiteLocalConfigService::dataCatalogs(void) const { + std::vector const &SiteLocalConfigService::trivialDataCatalogs() const { if (!m_connected) { - //throw cms::Exception("Incomplete configuration") - // << "Valid site-local-config not found at " << m_url; - // Return PoolFileCatalog.xml for now static std::vector const tmp{"file:PoolFileCatalog.xml"}; return tmp; } - if (m_dataCatalogs.empty()) { - throw cms::Exception("Incomplete configuration") << "Did not find catalogs in event-data section in " << m_url; + if (m_trivialDataCatalogs.empty()) { + cms::Exception ex("SiteLocalConfigService"); + ex << "Did not find catalogs in event-data section in " << m_url; + ex.addContext("edm::SiteLocalConfigService::trivialDataCatalogs()"); + throw ex; } + return m_trivialDataCatalogs; + } + + std::vector const &SiteLocalConfigService::dataCatalogs() const { + if (!m_connected) { + cms::Exception ex("SiteLocalConfigService"); + ex << "Incomplete configuration. Valid site-local-config not found at " << m_url; + ex.addContext("edm::SiteLocalConfigService::dataCatalogs()"); + throw ex; + } + if (m_dataCatalogs.empty()) { + cms::Exception ex("SiteLocalConfigService"); + ex << "Did not find catalogs in data-access section in " << m_url; + ex.addContext("edm::SiteLocalConfigService::dataCatalogs()"); + throw ex; + } return m_dataCatalogs; } + std::filesystem::path const SiteLocalConfigService::storageDescriptionPath( + edm::CatalogAttributes const &aDataCatalog) const { + std::string siteconfig_path = std::string(std::getenv("SITECONFIG_PATH")); + std::filesystem::path filename_storage; + //not a cross site use local path given in SITECONFIG_PATH + if (aDataCatalog.site == aDataCatalog.storageSite) { + //it is a site (no defined subSite), use local path given in SITECONFIG_PATH + if (aDataCatalog.subSite.empty()) + filename_storage = siteconfig_path; + //it is a subsite, move one level up + else + filename_storage = siteconfig_path + "/.."; + } else { //cross site + //it is a site (no defined subSite), move one level up + if (aDataCatalog.subSite.empty()) + filename_storage = siteconfig_path + "/../" + aDataCatalog.storageSite; + //it is a subsite, move two levels up + else + filename_storage = siteconfig_path + "/../../" + aDataCatalog.storageSite; + } + filename_storage /= "storage.json"; + try { + filename_storage = std::filesystem::canonical(filename_storage); + } catch (std::exception &e) { + cms::Exception ex("SiteLocalConfigService"); + ex << "Fail to convert path to the storage description, " << filename_storage.string() + << ", to the canonical absolute path" + << ". Path exists?"; + ex.addContext("edm::SiteLocalConfigService::storageDescriptionPath()"); + throw ex; + } + return filename_storage; + } + std::string const SiteLocalConfigService::frontierConnect(std::string const &servlet) const { if (!m_connected) { throw cms::Exception("Incomplete configuration") << "Valid site-local-config not found at " << m_url; @@ -268,10 +319,27 @@ namespace edm { } std::string const &SiteLocalConfigService::siteName() const { return m_siteName; } + std::string const &SiteLocalConfigService::subSiteName() const { return m_subSiteName; } bool SiteLocalConfigService::useLocalConnectString() const { return m_useLocalConnectString; } std::string const &SiteLocalConfigService::localConnectPrefix() const { return m_localConnectPrefix; } std::string const &SiteLocalConfigService::localConnectSuffix() const { return m_localConnectSuffix; } + void SiteLocalConfigService::getCatalog(tinyxml2::XMLElement const &cat, std::string site, std::string subSite) { + edm::CatalogAttributes aCatalog; + aCatalog.site = site; + aCatalog.subSite = subSite; + auto tmp_site = std::string(safe(cat.Attribute("site"))); + //no site attribute in the data catalog defined in , so storage site is from block of site_local_config.xml, which is the input parameter "site" of this method + if (tmp_site.empty()) + aCatalog.storageSite = site; + //now storage site is explicitly defined in + else + aCatalog.storageSite = tmp_site; + aCatalog.volume = std::string(safe(cat.Attribute("volume"))); + aCatalog.protocol = std::string(safe(cat.Attribute("protocol"))); + m_dataCatalogs.push_back(aCatalog); + } + void SiteLocalConfigService::parse(std::string const &url) { tinyxml2::XMLDocument doc; auto loadErr = doc.LoadFile(url.c_str()); @@ -282,6 +350,7 @@ namespace edm { // The Site Config has the following format // // + // // // // @@ -307,142 +376,178 @@ namespace edm { // // // + auto rootElement = doc.RootElement(); for (auto site = rootElement->FirstChildElement("site"); site != nullptr; site = site->NextSiblingElement("site")) { + auto subSite = site->FirstChildElement("subsite"); + // Parse the site name m_siteName = safe(site->Attribute("name")); + m_subSiteName = std::string(); + if (subSite) { + //check to make sure subSite has no children + auto subSite_first_child = subSite->FirstChild(); + if (subSite_first_child) { + cms::Exception ex("SiteLocalConfigService"); + ex << "Invalid site-local-config.xml. Subsite node has children!"; + ex.addContext("edm::SiteLocalConfigService::parse()"); + throw ex; + } + m_subSiteName = safe(subSite->Attribute("name")); + } // Parsing of the event data section - { - auto eventData = site->FirstChildElement("event-data"); - if (eventData) { - auto catalog = eventData->FirstChildElement("catalog"); - if (catalog) { - m_dataCatalogs.push_back(safe(catalog->Attribute("url"))); + auto eventData = site->FirstChildElement("event-data"); + if (eventData) { + auto catalog = eventData->FirstChildElement("catalog"); + if (catalog) { + m_trivialDataCatalogs.push_back(safe(catalog->Attribute("url"))); + catalog = catalog->NextSiblingElement("catalog"); + while (catalog) { + m_trivialDataCatalogs.push_back(safe(catalog->Attribute("url"))); catalog = catalog->NextSiblingElement("catalog"); - while (catalog) { - m_dataCatalogs.push_back(safe(catalog->Attribute("url"))); - catalog = catalog->NextSiblingElement("catalog"); - } } - auto rfiotype = eventData->FirstChildElement("rfiotype"); - if (rfiotype) { - m_rfioType = safe(rfiotype->Attribute("value")); + } + auto rfiotype = eventData->FirstChildElement("rfiotype"); + if (rfiotype) { + m_rfioType = safe(rfiotype->Attribute("value")); + } + } + + //data-access + //let store catalog entry as: SITE,SUBSITE,STORAGE_SITE,VOLUME,PROTOCOL + // SITE: from element + // SUBSITE: from element. SUBSITE=SITE for site + // STORAGE_SITE, VOLUME and PROTOCOL: from in . If "site" attribute is not defined inside , STORAGE_SITE is SITE + //Therefore + //1. if STORAGE_SITE = SITE, use local storage.json since STORAGE_SITE is not a cross site + //2. if SUBSITE is empty, this is a site. Otherwise, this is a subsite. These are used to define the path to locate the storage.json in FileLocator. This path is provided by storageDescriptionPath() method of this class. + //get data-access + auto dataAccess = site->FirstChildElement("data-access"); + if (dataAccess) { + //get catalogs + auto catalog = dataAccess->FirstChildElement("catalog"); + if (catalog) { + //add all info for the first catlog here + getCatalog(*catalog, m_siteName, m_subSiteName); + //get next catlog + catalog = catalog->NextSiblingElement("catalog"); + while (catalog) { + //add all info for the current catlog here + getCatalog(*catalog, m_siteName, m_subSiteName); + //get next catlog + catalog = catalog->NextSiblingElement("catalog"); } } } // Parsing of the calib-data section - { - auto calibData = site->FirstChildElement("calib-data"); + auto calibData = site->FirstChildElement("calib-data"); - if (calibData) { - auto frontierConnect = calibData->FirstChildElement("frontier-connect"); + if (calibData) { + auto frontierConnect = calibData->FirstChildElement("frontier-connect"); + if (frontierConnect) { + m_frontierConnect = _toParenString(*frontierConnect); + } + auto localConnect = calibData->FirstChildElement("local-connect"); + if (localConnect) { if (frontierConnect) { - m_frontierConnect = _toParenString(*frontierConnect); + throw cms::Exception("Illegal site local configuration") + << "It is illegal to include both frontier-connect and local-connect in the same XML file"; } - auto localConnect = calibData->FirstChildElement("local-connect"); - if (localConnect) { - if (frontierConnect) { - throw cms::Exception("Illegal site local configuration") - << "It is illegal to include both frontier-connect and local-connect in the same XML file"; - } - m_useLocalConnectString = true; - auto connectString = localConnect->FirstChildElement("connectString"); - if (connectString) { - m_localConnectPrefix = safe(connectString->Attribute("prefix")); - m_localConnectSuffix = safe(connectString->Attribute("suffix")); - } + m_useLocalConnectString = true; + auto connectString = localConnect->FirstChildElement("connectString"); + if (connectString) { + m_localConnectPrefix = safe(connectString->Attribute("prefix")); + m_localConnectSuffix = safe(connectString->Attribute("suffix")); } } } // Parsing of the source config section - { - auto sourceConfig = site->FirstChildElement("source-config"); + auto sourceConfig = site->FirstChildElement("source-config"); - if (sourceConfig) { - auto cacheTempDir = sourceConfig->FirstChildElement("cache-temp-dir"); + if (sourceConfig) { + auto cacheTempDir = sourceConfig->FirstChildElement("cache-temp-dir"); - if (cacheTempDir) { - m_cacheTempDir = safe(cacheTempDir->Attribute("name")); - m_cacheTempDirPtr = &m_cacheTempDir; - } + if (cacheTempDir) { + m_cacheTempDir = safe(cacheTempDir->Attribute("name")); + m_cacheTempDirPtr = &m_cacheTempDir; + } - auto cacheMinFree = sourceConfig->FirstChildElement("cache-min-free"); + auto cacheMinFree = sourceConfig->FirstChildElement("cache-min-free"); - if (cacheMinFree) { - //TODO what did xerces do if it couldn't convert? - m_cacheMinFree = cacheMinFree->DoubleAttribute("value"); - m_cacheMinFreePtr = &m_cacheMinFree; - } + if (cacheMinFree) { + //TODO what did xerces do if it couldn't convert? + m_cacheMinFree = cacheMinFree->DoubleAttribute("value"); + m_cacheMinFreePtr = &m_cacheMinFree; + } - auto cacheHint = sourceConfig->FirstChildElement("cache-hint"); + auto cacheHint = sourceConfig->FirstChildElement("cache-hint"); - if (cacheHint) { - m_cacheHint = safe(cacheHint->Attribute("value")); - m_cacheHintPtr = &m_cacheHint; - } + if (cacheHint) { + m_cacheHint = safe(cacheHint->Attribute("value")); + m_cacheHintPtr = &m_cacheHint; + } - auto cloneCacheHint = sourceConfig->FirstChildElement("clone-cache-hint"); + auto cloneCacheHint = sourceConfig->FirstChildElement("clone-cache-hint"); - if (cloneCacheHint) { - m_cloneCacheHint = safe(cloneCacheHint->Attribute("value")); - m_cloneCacheHintPtr = &m_cloneCacheHint; - } + if (cloneCacheHint) { + m_cloneCacheHint = safe(cloneCacheHint->Attribute("value")); + m_cloneCacheHintPtr = &m_cloneCacheHint; + } - auto readHint = sourceConfig->FirstChildElement("read-hint"); + auto readHint = sourceConfig->FirstChildElement("read-hint"); - if (readHint) { - m_readHint = safe(readHint->Attribute("value")); - m_readHintPtr = &m_readHint; - } + if (readHint) { + m_readHint = safe(readHint->Attribute("value")); + m_readHintPtr = &m_readHint; + } - auto ttreeCacheSize = sourceConfig->FirstChildElement("ttree-cache-size"); + auto ttreeCacheSize = sourceConfig->FirstChildElement("ttree-cache-size"); - if (ttreeCacheSize) { - m_ttreeCacheSize = ttreeCacheSize->UnsignedAttribute("value"); - m_ttreeCacheSizePtr = &m_ttreeCacheSize; - } + if (ttreeCacheSize) { + m_ttreeCacheSize = ttreeCacheSize->UnsignedAttribute("value"); + m_ttreeCacheSizePtr = &m_ttreeCacheSize; + } - auto timeout = sourceConfig->FirstChildElement("timeout-in-seconds"); + auto timeout = sourceConfig->FirstChildElement("timeout-in-seconds"); - if (timeout) { - m_timeout = timeout->UnsignedAttribute("value"); - m_timeoutPtr = &m_timeout; - } + if (timeout) { + m_timeout = timeout->UnsignedAttribute("value"); + m_timeoutPtr = &m_timeout; + } - auto statsDest = sourceConfig->FirstChildElement("statistics-destination"); + auto statsDest = sourceConfig->FirstChildElement("statistics-destination"); - if (statsDest) { - m_statisticsDestination = safe(statsDest->Attribute("endpoint")); - if (m_statisticsDestination.empty()) { - m_statisticsDestination = safe(statsDest->Attribute("name")); - } - std::string tmpStatisticsInfo = safe(statsDest->Attribute("info")); - boost::split(m_statisticsInfo, tmpStatisticsInfo, boost::is_any_of("\t ,")); - m_statisticsInfoAvail = !tmpStatisticsInfo.empty(); + if (statsDest) { + m_statisticsDestination = safe(statsDest->Attribute("endpoint")); + if (m_statisticsDestination.empty()) { + m_statisticsDestination = safe(statsDest->Attribute("name")); } + std::string tmpStatisticsInfo = safe(statsDest->Attribute("info")); + boost::split(m_statisticsInfo, tmpStatisticsInfo, boost::is_any_of("\t ,")); + m_statisticsInfoAvail = !tmpStatisticsInfo.empty(); + } - auto prefetching = sourceConfig->FirstChildElement("prefetching"); + auto prefetching = sourceConfig->FirstChildElement("prefetching"); - if (prefetching) { - m_enablePrefetching = prefetching->BoolAttribute("value"); - m_enablePrefetchingPtr = &m_enablePrefetching; - } + if (prefetching) { + m_enablePrefetching = prefetching->BoolAttribute("value"); + m_enablePrefetchingPtr = &m_enablePrefetching; + } - auto nativeProtocol = sourceConfig->FirstChildElement("native-protocols"); + auto nativeProtocol = sourceConfig->FirstChildElement("native-protocols"); - if (nativeProtocol) { - for (auto child = nativeProtocol->FirstChildElement(); child != nullptr; - child = child->NextSiblingElement()) { - m_nativeProtocols.push_back(safe(child->Attribute("prefix"))); - } - m_nativeProtocolsPtr = &m_nativeProtocols; + if (nativeProtocol) { + for (auto child = nativeProtocol->FirstChildElement(); child != nullptr; + child = child->NextSiblingElement()) { + m_nativeProtocols.push_back(safe(child->Attribute("prefix"))); } + m_nativeProtocolsPtr = &m_nativeProtocols; } } } diff --git a/FWCore/Services/src/SiteLocalConfigService.h b/FWCore/Services/src/SiteLocalConfigService.h index 283209eebe482..55a500e5346eb 100644 --- a/FWCore/Services/src/SiteLocalConfigService.h +++ b/FWCore/Services/src/SiteLocalConfigService.h @@ -3,8 +3,10 @@ /////////////////////////////////////////////////////////////////////// // -// dataCatalogs() returns multiple data catalogs in site-local-config.xml -// +// dataCatalogs(unsigned catType) returns multiple data catalogs in site-local-config.xml. +// catType=TrivialCatalog: returns trivial catalogs defined in blocks +// catType=RucioCatalog: returns catalogs defined in blocks +// (the "enum" is defined in FWCore/Catalog/interface/SiteLocalConfig.h) /////////////////////////////////////////////////////////////////////// //<<<<<< INCLUDES >>>>>> #include @@ -12,6 +14,8 @@ #include #include "FWCore/Catalog/interface/SiteLocalConfig.h" #include "FWCore/Utilities/interface/propagate_const.h" +#include "tinyxml2.h" + //<<<<<< PUBLIC DEFINES >>>>>> //<<<<<< PUBLIC CONSTANTS >>>>>> //<<<<<< PUBLIC TYPES >>>>>> @@ -26,11 +30,14 @@ namespace edm { namespace edm { class ConfigurationDescriptions; namespace service { + class SiteLocalConfigService : public SiteLocalConfig { public: explicit SiteLocalConfigService(ParameterSet const& pset); - std::vector const& dataCatalogs(void) const override; + std::vector const& trivialDataCatalogs() const override; + std::vector const& dataCatalogs() const override; + std::filesystem::path const storageDescriptionPath(edm::CatalogAttributes const& aDataCatalog) const override; std::string const lookupCalibConnect(std::string const& input) const override; std::string const rfioType(void) const override; @@ -47,6 +54,7 @@ namespace edm { struct addrinfo const* statisticsDestination() const override; std::set const* statisticsInfo() const override; std::string const& siteName() const override; + std::string const& subSiteName() const override; bool useLocalConnectString() const override; std::string const& localConnectPrefix() const override; std::string const& localConnectSuffix() const override; @@ -58,11 +66,13 @@ namespace edm { static void fillDescriptions(ConfigurationDescriptions& descriptions); private: + void getCatalog(tinyxml2::XMLElement const& cat, std::string site, std::string subSite); void parse(std::string const& url); void computeStatisticsDestination(); std::string const frontierConnect(std::string const& servlet) const; std::string m_url; - std::vector m_dataCatalogs; + std::vector m_trivialDataCatalogs; + std::vector m_dataCatalogs; std::string m_frontierConnect; std::string m_rfioType; bool m_connected; @@ -91,6 +101,7 @@ namespace edm { std::set m_statisticsInfo; bool m_statisticsInfoAvail; std::string m_siteName; + std::string m_subSiteName; bool m_useLocalConnectString = false; std::string m_localConnectPrefix; std::string m_localConnectSuffix; diff --git a/FWCore/Services/test/full-site-local-config.testfile b/FWCore/Services/test/full-site-local-config.testfile index 78d7ebbf51656..3f13645abd491 100644 --- a/FWCore/Services/test/full-site-local-config.testfile +++ b/FWCore/Services/test/full-site-local-config.testfile @@ -1,8 +1,12 @@ + + + + diff --git a/FWCore/Services/test/no-source-site-local-config.testfile b/FWCore/Services/test/no-source-site-local-config.testfile index 28a0e422045ac..1788db27b33d4 100644 --- a/FWCore/Services/test/no-source-site-local-config.testfile +++ b/FWCore/Services/test/no-source-site-local-config.testfile @@ -1,8 +1,12 @@ + + + + diff --git a/FWCore/Services/test/source-site-local-config.testfile b/FWCore/Services/test/source-site-local-config.testfile index 426539fdd7c30..92d471bec7130 100644 --- a/FWCore/Services/test/source-site-local-config.testfile +++ b/FWCore/Services/test/source-site-local-config.testfile @@ -1,8 +1,12 @@ + + + + diff --git a/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc b/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc index 3249196660f5c..796d23785e507 100644 --- a/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc +++ b/FWCore/Services/test/test_catch2_SiteLocalConfigService.cc @@ -1,3 +1,4 @@ +#include "FWCore/Catalog/interface/SiteLocalConfig.h" #include "FWCore/Services/src/SiteLocalConfigService.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/Utilities/interface/Exception.h" @@ -24,7 +25,13 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") { edm::service::SiteLocalConfigService slc(pset); - CHECK(slc.dataCatalogs()[0] == "trivialcatalog_file:/dummy/storage.xml?protocol=dcap"); + CHECK(slc.trivialDataCatalogs()[0] == "trivialcatalog_file:/dummy/storage.xml?protocol=dcap"); + edm::CatalogAttributes tmp("DUMMY", "DUMMY_SUB_SITE", "DUMMY_CROSS_SITE", "CMSXrootdFederation", "XRootD"); + CHECK(slc.dataCatalogs()[0].site == tmp.site); + CHECK(slc.dataCatalogs()[0].subSite == tmp.subSite); + CHECK(slc.dataCatalogs()[0].storageSite == tmp.storageSite); + CHECK(slc.dataCatalogs()[0].volume == tmp.volume); + CHECK(slc.dataCatalogs()[0].protocol == tmp.protocol); REQUIRE(slc.sourceCacheTempDir() != nullptr); CHECK(*slc.sourceCacheTempDir() == "/a/b/c"); CHECK(slc.sourceCacheMinFree() == nullptr); @@ -47,6 +54,7 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") { CHECK(slc.statisticsInfo()->size() == 1); CHECK(slc.statisticsInfo()->find("dn") != slc.statisticsInfo()->end()); CHECK(slc.siteName() == "DUMMY"); + CHECK(slc.subSiteName() == "DUMMY_SUB_SITE"); REQUIRE(slc.useLocalConnectString() == true); REQUIRE(slc.localConnectPrefix() == "Test:Prefix"); REQUIRE(slc.localConnectSuffix() == "Test.Suffix"); @@ -73,7 +81,13 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") { edm::service::SiteLocalConfigService slc(pset); - CHECK(slc.dataCatalogs()[0] == "trivialcatalog_file:/dummy/storage.xml?protocol=dcap"); + CHECK(slc.trivialDataCatalogs()[0] == "trivialcatalog_file:/dummy/storage.xml?protocol=dcap"); + edm::CatalogAttributes tmp("DUMMY", "DUMMY_SUB_SITE", "DUMMY_CROSS_SITE", "CMSXrootdFederation", "XRootD"); + CHECK(slc.dataCatalogs()[0].site == tmp.site); + CHECK(slc.dataCatalogs()[0].subSite == tmp.subSite); + CHECK(slc.dataCatalogs()[0].storageSite == tmp.storageSite); + CHECK(slc.dataCatalogs()[0].volume == tmp.volume); + CHECK(slc.dataCatalogs()[0].protocol == tmp.protocol); REQUIRE(slc.sourceCacheTempDir() != nullptr); CHECK(*slc.sourceCacheTempDir() == "/a/d"); REQUIRE(slc.sourceCacheMinFree() != nullptr); @@ -97,6 +111,7 @@ TEST_CASE("Test SiteLocalConfigService", "[sitelocalconfig]") { CHECK(slc.statisticsInfo()->size() == 1); CHECK(slc.statisticsInfo()->find("nodn") != slc.statisticsInfo()->end()); CHECK(slc.siteName() == "DUMMY"); + CHECK(slc.subSiteName() == "DUMMY_SUB_SITE"); REQUIRE(slc.useLocalConnectString() == false); REQUIRE(slc.localConnectPrefix() == "OverridePrefix"); REQUIRE(slc.localConnectSuffix() == "OverrideSuffix"); diff --git a/FWCore/Services/test/test_sitelocalconfig.sh b/FWCore/Services/test/test_sitelocalconfig.sh index c174dd594e1c5..01f7ba4efc91f 100755 --- a/FWCore/Services/test/test_sitelocalconfig.sh +++ b/FWCore/Services/test/test_sitelocalconfig.sh @@ -6,10 +6,11 @@ function die { echo $1: status $2 ; exit $2; } mkdir -p ${CMSSW_BASE}/test/SITECONF mkdir -p ${CMSSW_BASE}/test/SITECONF/local mkdir -p ${CMSSW_BASE}/test/SITECONF/local/JobConfig -cp ${LOCAL_TEST_DIR}/no-source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml +export SITECONFIG_PATH=${CMSSW_BASE}/test/SITECONF/local + +cp ${LOCAL_TEST_DIR}/no-source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml F1=${LOCAL_TEST_DIR}/test_sitelocalconfig_no_source_cfg.py -export CMS_PATH=${CMSSW_BASE}/test (cmsRun $F1 ) || die "Failure using $F1" $? cp ${LOCAL_TEST_DIR}/source-site-local-config.testfile ${CMSSW_BASE}/test/SITECONF/local/JobConfig/site-local-config.xml