Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix chaining in rules and unify prefix and rules of protocols #42530

Merged
merged 1 commit into from
Aug 18, 2023
Merged

Fix chaining in rules and unify prefix and rules of protocols #42530

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion FWCore/Catalog/interface/FileLocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ namespace edm {
typedef std::map<std::string, Rules> ProtocolRules;

void init_trivialCatalog(std::string const& catUrl, unsigned iCatalog);

void parseRuleTrivialCatalog(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules);
//using data-access
void init(edm::CatalogAttributes const& input_dataCatalog,
Expand All @@ -68,9 +69,11 @@ namespace edm {

std::string m_fileType;
std::string m_filename;
//TFC allows more than one protocols provided in a catalog, separated by a comma
//In new Rucio storage description, only one protocol is provided in a catalog
//This variable can be simplified in the future
std::vector<std::string> m_protocols;
std::string m_destination;
std::string m_prefix;
};
} // namespace edm

Expand Down
46 changes: 22 additions & 24 deletions FWCore/Catalog/src/FileLocator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,14 +70,6 @@ namespace edm {
//check if input is an authentic LFN
if (input.compare(0, 7, kLFNPrefix) != 0)
return out;
//use prefix in the protocol
if (!m_prefix.empty()) {
out = m_prefix + "/" + input;
if (input[0] == '/')
out = m_prefix + input;
return out;
}
//no prefix in the protocol, use rule
for (size_t pi = 0, pe = m_protocols.size(); pi != pe; ++pi) {
out = applyRules(rules, m_protocols[pi], m_destination, direct, input);
if (!out.empty()) {
Expand Down Expand Up @@ -118,11 +110,12 @@ namespace edm {
}
auto const pathMatchRegexp = storageRule.second.get<std::string>("lfn");
auto const result = storageRule.second.get<std::string>("pfn");
auto const chain = storageRule.second.get("chain", kEmptyString);
Rule rule;
rule.pathMatch.assign(pathMatchRegexp);
rule.destinationMatch.assign(".*");
rule.result = result;
rule.chain = "";
rule.chain = chain;
rules[protocol].emplace_back(std::move(rule));
}

Expand Down Expand Up @@ -293,22 +286,27 @@ namespace edm {

std::string protName = found_protocol->second.get("protocol", kEmptyString);
m_protocols.push_back(protName);
m_prefix = found_protocol->second.get("prefix", kEmptyString);
if (m_prefix == kEmptyString) {
//get rules
if (found_protocol->second.find("rules") == found_protocol->second.not_found()) {
cms::Exception ex("FileCatalog");
ex << "protocol must contain either a prefix or rules, "
<< "neither found for protocol \"" << aCatalog.protocol << "\" for the storage site \""
<< aCatalog.storageSite << "\" and volume \"" << aCatalog.volume
<< "\" in storage.json. Check site-local-config.xml <data-access> and storage.json";
ex.addContext("edm::FileLocator:init()");
throw ex;
}
const pt::ptree& rules = found_protocol->second.find("rules")->second;

//store all prefixes and rules to m_directRules. We need to do this so that "applyRules" can find the rule in case chaining is used
//loop over protocols
for (pt::ptree::value_type const& protocol : protocols) {
std::string protName = protocol.second.get("protocol", kEmptyString);
//loop over rules
for (pt::ptree::value_type const& storageRule : rules) {
parseRule(storageRule, protName, m_directRules);
std::string prefixTmp = protocol.second.get("prefix", kEmptyString);
if (prefixTmp == kEmptyString) {
const pt::ptree& rules = protocol.second.find("rules")->second;
for (pt::ptree::value_type const& storageRule : rules) {
parseRule(storageRule, protName, m_directRules);
}
}
//now convert prefix to a rule and save it
else {
Rule rule;
rule.pathMatch.assign("/?(.*)");
rule.destinationMatch.assign(".*");
rule.result = prefixTmp + "/$1";
rule.chain = kEmptyString;
m_directRules[protName].emplace_back(std::move(rule));
}
}
}
Expand Down
42 changes: 38 additions & 4 deletions FWCore/Catalog/test/FileLocator_t.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,13 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
aCatalog.volume = "FNAL_dCache_EOS";
aCatalog.protocol = "XRootD";
tmp.push_back(aCatalog);
//catalog for testing chained "rules"
aCatalog.site = "T1_US_FNAL";
aCatalog.subSite = "T1_US_FNAL";
aCatalog.storageSite = "T1_US_FNAL";
aCatalog.volume = "FNAL_dCache_EOS";
aCatalog.protocol = "root";
tmp.push_back(aCatalog);

//create the services
edm::ServiceToken tempToken(
Expand All @@ -81,15 +88,19 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"

SECTION("prefix") {
edm::ServiceRegistry::Operate operate(tempToken);
edm::CatalogAttributes tmp_cat; //empty catalog
edm::FileLocator fl(tmp_cat, 0, full_file_name); //use the first catalog provided by site local config
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the first catalog provided by site local config
edm::FileLocator fl(tmp_cat, 0, full_file_name);
CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" ==
fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
}
SECTION("rule") {
edm::ServiceRegistry::Operate operate(tempToken);
edm::CatalogAttributes tmp_cat; //empty catalog
edm::FileLocator fl(tmp_cat, 1, full_file_name); //use the second catalog provided by site local config
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the second catalog provided by site local config
edm::FileLocator fl(tmp_cat, 1, full_file_name);
const std::array<const char*, 7> lfn = {{"/bha/bho",
"bha",
"file:bha",
Expand All @@ -103,6 +114,29 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
}
}
SECTION("chainedrule") {
edm::ServiceRegistry::Operate operate(tempToken);
//empty catalog
edm::CatalogAttributes tmp_cat;
//use the third catalog provided by site local config above
edm::FileLocator fl(tmp_cat, 2, full_file_name);
const std::array<const char*, 7> lfn = {{"/bha/bho",
"bha",
"file:bha",
"file:/bha/bho",
"/castor/cern.ch/cms/bha/bho",
"rfio:/castor/cern.ch/cms/bha/bho",
"rfio:/bha/bho"}};
//one level chain between "root" and "second" protocols (see storage.json)
CHECK("root://host.domain//pnfs/cms/store/group/bha/bho" ==
fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
//two levels chain between "root", "second" and "first" (see storage.json)
CHECK("root://host.domain//pnfs/cms/store/user/AAA/bho" ==
fl.pfn("/store/user/aaa/bho", edm::CatalogType::RucioCatalog));
for (auto file : lfn) {
CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
}
}
}

TEST_CASE("FileLocator", "[filelocator]") {
Expand Down
34 changes: 33 additions & 1 deletion FWCore/Catalog/test/storage.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,39 @@
"pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
}
]
}
},
{
"protocol": "first",
"rules": [
{ "lfn": "/+store/user/aaa/(.*)",
"pfn": "/store/user/AAA/$1"
},
{ "lfn": "/+(.*)",
"pfn": "$1"
}
]
},
{
"protocol": "second",
"rules": [
{ "lfn": "/+store/user/(.*)",
"pfn": "/cms/store/user/$1",
"chain": "first"
},
{ "lfn": "/+store/(.*)",
"pfn": "/cms/store/$1"
}
]
},
{
"protocol": "root",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "root://host.domain//pnfs/$1",
"chain": "second"
}
]
}
],
"type": "DISK",
"rse": "T1_US_FNAL_Disk",
Expand Down