Skip to content

Commit

Permalink
Merge pull request #42530 from nhduongvn/fix-chaining-in-rules
Browse files Browse the repository at this point in the history 
Fix chaining in rules and unify prefix and rules of protocols
  • Loading branch information
@cmsbuild
cmsbuild authored Aug 18, 2023
2 parents 1d77309 + 000b6fb commit 9a572f6
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 30 deletions.
5 changes: 4 additions & 1 deletion FWCore/Catalog/interface/FileLocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ namespace edm {
typedef std::map<std::string, Rules> ProtocolRules;

void init_trivialCatalog(std::string const& catUrl, unsigned iCatalog);

void parseRuleTrivialCatalog(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules);
//using data-access
void init(edm::CatalogAttributes const& input_dataCatalog,
Expand All @@ -68,9 +69,11 @@ namespace edm {

std::string m_fileType;
std::string m_filename;
//TFC allows more than one protocols provided in a catalog, separated by a comma
//In new Rucio storage description, only one protocol is provided in a catalog
//This variable can be simplified in the future
std::vector<std::string> m_protocols;
std::string m_destination;
std::string m_prefix;
};
} // namespace edm

Expand Down
46 changes: 22 additions & 24 deletions FWCore/Catalog/src/FileLocator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,6 @@ namespace edm {
//check if input is an authentic LFN
if (input.compare(0, 7, kLFNPrefix) != 0)
return out;
//use prefix in the protocol
if (!m_prefix.empty()) {
out = m_prefix + "/" + input;
if (input[0] == '/')
out = m_prefix + input;
return out;
}
//no prefix in the protocol, use rule
for (size_t pi = 0, pe = m_protocols.size(); pi != pe; ++pi) {
out = applyRules(rules, m_protocols[pi], m_destination, direct, input);
if (!out.empty()) {
Expand Down Expand Up @@ -118,11 +110,12 @@ namespace edm {
}
auto const pathMatchRegexp = storageRule.second.get<std::string>("lfn");
auto const result = storageRule.second.get<std::string>("pfn");
auto const chain = storageRule.second.get("chain", kEmptyString);
Rule rule;
rule.pathMatch.assign(pathMatchRegexp);
rule.destinationMatch.assign(".*");
rule.result = result;
rule.chain = "";
rule.chain = chain;
rules[protocol].emplace_back(std::move(rule));
}

Expand Down Expand Up @@ -293,22 +286,27 @@ namespace edm {

std::string protName = found_protocol->second.get("protocol", kEmptyString);
m_protocols.push_back(protName);
m_prefix = found_protocol->second.get("prefix", kEmptyString);
if (m_prefix == kEmptyString) {
//get rules
if (found_protocol->second.find("rules") == found_protocol->second.not_found()) {
cms::Exception ex("FileCatalog");
ex << "protocol must contain either a prefix or rules, "
<< "neither found for protocol \"" << aCatalog.protocol << "\" for the storage site \""
<< aCatalog.storageSite << "\" and volume \"" << aCatalog.volume
<< "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
ex.addContext("edm::FileLocator:init()");
throw ex;
}
const pt::ptree& rules = found_protocol->second.find("rules")->second;

//store all prefixes and rules to m_directRules. We need to do this so that "applyRules" can find the rule in case chaining is used
//loop over protocols
for (pt::ptree::value_type const& protocol : protocols) {
std::string protName = protocol.second.get("protocol", kEmptyString);
//loop over rules
for (pt::ptree::value_type const& storageRule : rules) {
parseRule(storageRule, protName, m_directRules);
std::string prefixTmp = protocol.second.get("prefix", kEmptyString);
if (prefixTmp == kEmptyString) {
const pt::ptree& rules = protocol.second.find("rules")->second;
for (pt::ptree::value_type const& storageRule : rules) {
parseRule(storageRule, protName, m_directRules);
}
}
//now convert prefix to a rule and save it
else {
Rule rule;
rule.pathMatch.assign("/?(.*)");
rule.destinationMatch.assign(".*");
rule.result = prefixTmp + "/$1";
rule.chain = kEmptyString;
m_directRules[protName].emplace_back(std::move(rule));
}
}
}
Expand Down
42 changes: 38 additions & 4 deletions FWCore/Catalog/test/FileLocator_t.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
aCatalog.volume = "FNAL_dCache_EOS";
aCatalog.protocol = "XRootD";
tmp.push_back(aCatalog);
//catalog for testing chained "rules"
aCatalog.site = "T1_US_FNAL";
aCatalog.subSite = "T1_US_FNAL";
aCatalog.storageSite = "T1_US_FNAL";
aCatalog.volume = "FNAL_dCache_EOS";
aCatalog.protocol = "root";
tmp.push_back(aCatalog);

//create the services
edm::ServiceToken tempToken(
Expand All @@ -81,15 +88,19 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"

SECTION("prefix") {
edm::ServiceRegistry::Operate operate(tempToken);
edm::CatalogAttributes tmp_cat; //empty catalog
edm::FileLocator fl(tmp_cat, 0, full_file_name); //use the first catalog provided by site local config
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the first catalog provided by site local config
edm::FileLocator fl(tmp_cat, 0, full_file_name);
CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" ==
fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
}
SECTION("rule") {
edm::ServiceRegistry::Operate operate(tempToken);
edm::CatalogAttributes tmp_cat; //empty catalog
edm::FileLocator fl(tmp_cat, 1, full_file_name); //use the second catalog provided by site local config
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the second catalog provided by site local config
edm::FileLocator fl(tmp_cat, 1, full_file_name);
const std::array<const char*, 7> lfn = {{"/bha/bho",
"bha",
"file:bha",
Expand All @@ -103,6 +114,29 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
}
}
SECTION("chainedrule") {
edm::ServiceRegistry::Operate operate(tempToken);
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the third catalog provided by site local config above
edm::FileLocator fl(tmp_cat, 2, full_file_name);
const std::array<const char*, 7> lfn = {{"/bha/bho",
"bha",
"file:bha",
"file:/bha/bho",
"/castor/cern.ch/cms/bha/bho",
"rfio:/castor/cern.ch/cms/bha/bho",
"rfio:/bha/bho"}};
//one level chain between "root" and "second" protocols (see storage.json)
CHECK("root://host.domain//pnfs/cms/store/group/bha/bho" ==
fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
//two levels chain between "root", "second" and "first" (see storage.json)
CHECK("root://host.domain//pnfs/cms/store/user/AAA/bho" ==
fl.pfn("/store/user/aaa/bho", edm::CatalogType::RucioCatalog));
for (auto file : lfn) {
CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
}
}
}

TEST_CASE("FileLocator", "[filelocator]") {
Expand Down
34 changes: 33 additions & 1 deletion FWCore/Catalog/test/storage.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,39 @@
"pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
}
]
}
},
{
"protocol": "first",
"rules": [
{ "lfn": "/+store/user/aaa/(.*)",
"pfn": "/store/user/AAA/$1"
},
{ "lfn": "/+(.*)",
"pfn": "$1"
}
]
},
{
"protocol": "second",
"rules": [
{ "lfn": "/+store/user/(.*)",
"pfn": "/cms/store/user/$1",
"chain": "first"
},
{ "lfn": "/+store/(.*)",
"pfn": "/cms/store/$1"
}
]
},
{
"protocol": "root",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "root://host.domain//pnfs/$1",
"chain": "second"
}
]
}
],
"type": "DISK",
"rse": "T1_US_FNAL_Disk",
Expand Down

0 comments on commit 9a572f6

Please sign in to comment.