Matti reviews 1

cms-sw · Aug 11, 2023 · 05b57b1 · 05b57b1
1 parent 6abadbf
commit 05b57b1
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 10 deletions.
diff --git a/FWCore/Catalog/interface/FileLocator.h b/FWCore/Catalog/interface/FileLocator.h
@@ -43,7 +43,6 @@ namespace edm {
 
     void init_trivialCatalog(std::string const& catUrl, unsigned iCatalog);
 
-    //void ruleBuilder(std::string prefix, Rule& rule);
     void parseRuleTrivialCatalog(tinyxml2::XMLElement* ruleNode, ProtocolRules& rules);
     //using data-access
     void init(edm::CatalogAttributes const& input_dataCatalog,
@@ -70,8 +69,10 @@ namespace edm {
 
     std::string m_fileType;
     std::string m_filename;
-    std::vector<std::string>
-        m_protocols;  //not sure why a vector is used while it seems this vector has only one element?
+    //TFC allows more than one protocols provided in a catalog, separated by a comma
+    //In new Rucio storage description, only one protocol is provided in a catalog
+    //This variable can be simplified in the future
+    std::vector<std::string> m_protocols;
     std::string m_destination;
   };
 }  // namespace edm

diff --git a/FWCore/Catalog/src/FileLocator.cc b/FWCore/Catalog/src/FileLocator.cc
@@ -1,5 +1,6 @@
 #include "FWCore/Catalog/interface/FileLocator.h"
 #include "FWCore/ServiceRegistry/interface/Service.h"
+#include "FWCore/Utilities/interface/Exception.h"
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/replace.hpp>
@@ -301,9 +302,9 @@ namespace edm {
       //now convert prefix to a rule and save it
       else {
         Rule rule;
-        rule.pathMatch.assign("(/.*)");  //require lfn start with "/"
+        rule.pathMatch.assign("/?(.*)");
         rule.destinationMatch.assign(".*");
-        rule.result = prefixTmp + "$1";
+        rule.result = prefixTmp + "/$1";
         rule.chain = kEmptyString;
         m_directRules[protName].emplace_back(std::move(rule));
       }

diff --git a/FWCore/Catalog/test/FileLocator_t.cpp b/FWCore/Catalog/test/FileLocator_t.cpp
@@ -67,6 +67,13 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
   aCatalog.volume = "FNAL_dCache_EOS";
   aCatalog.protocol = "XRootD";
   tmp.push_back(aCatalog);
+  //catalog for testing chained "rules"
+  aCatalog.site = "T1_US_FNAL";
+  aCatalog.subSite = "T1_US_FNAL";
+  aCatalog.storageSite = "T1_US_FNAL";
+  aCatalog.volume = "FNAL_dCache_EOS";
+  aCatalog.protocol = "root";
+  tmp.push_back(aCatalog);
 
   //create the services
   edm::ServiceToken tempToken(
@@ -81,15 +88,19 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
 
   SECTION("prefix") {
     edm::ServiceRegistry::Operate operate(tempToken);
-    edm::CatalogAttributes tmp_cat;                   //empty catalog
-    edm::FileLocator fl(tmp_cat, 0, full_file_name);  //use the first catalog provided by site local config
+    //empty catalog
+    edm::CatalogAttributes tmp_cat;
+    //use the first catalog provided by site local config
+    edm::FileLocator fl(tmp_cat, 0, full_file_name);
     CHECK("root://cmsxrootd.fnal.gov/store/group/bha/bho" ==
           fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
   }
   SECTION("rule") {
     edm::ServiceRegistry::Operate operate(tempToken);
-    edm::CatalogAttributes tmp_cat;                   //empty catalog
-    edm::FileLocator fl(tmp_cat, 1, full_file_name);  //use the second catalog provided by site local config
+    //empty catalog
+    edm::CatalogAttributes tmp_cat;
+    //use the second catalog provided by site local config
+    edm::FileLocator fl(tmp_cat, 1, full_file_name);
     const std::array<const char*, 7> lfn = {{"/bha/bho",
                                              "bha",
                                              "file:bha",
@@ -103,6 +114,29 @@ TEST_CASE("FileLocator with Rucio data catalog", "[filelocatorRucioDataCatalog]"
       CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
     }
   }
+  SECTION("chainedrule") {
+    edm::ServiceRegistry::Operate operate(tempToken);
+    //empty catalog
+    edm::CatalogAttributes tmp_cat;
+    //use the third catalog provided by site local config above
+    edm::FileLocator fl(tmp_cat, 2, full_file_name);
+    const std::array<const char*, 7> lfn = {{"/bha/bho",
+                                             "bha",
+                                             "file:bha",
+                                             "file:/bha/bho",
+                                             "/castor/cern.ch/cms/bha/bho",
+                                             "rfio:/castor/cern.ch/cms/bha/bho",
+                                             "rfio:/bha/bho"}};
+    //one level chain between "root" and "second" protocols (see storage.json)
+    CHECK("root://host.domain//pnfs/cms/store/group/bha/bho" ==
+          fl.pfn("/store/group/bha/bho", edm::CatalogType::RucioCatalog));
+    //two levels chain between "root", "second" and "first" (see storage.json)
+    CHECK("root://host.domain//pnfs/cms/store/user/AAA/bho" ==
+          fl.pfn("/store/user/aaa/bho", edm::CatalogType::RucioCatalog));
+    for (auto file : lfn) {
+      CHECK("" == fl.pfn(file, edm::CatalogType::RucioCatalog));
+    }
+  }
 }
 
 TEST_CASE("FileLocator", "[filelocator]") {

diff --git a/FWCore/Catalog/test/storage.json b/FWCore/Catalog/test/storage.json
@@ -13,7 +13,39 @@
                   "pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
                }
             ]
-         }
+         },
+         {
+            "protocol": "first",
+            "rules": [
+               { "lfn": "/+store/user/aaa/(.*)",
+                 "pfn": "/store/user/AAA/$1"
+               },
+               { "lfn": "/+(.*)",
+                 "pfn": "$1"
+               }
+            ]
+         },
+         {
+             "protocol": "second",
+             "rules": [
+              { "lfn": "/+store/user/(.*)",
+                "pfn": "/cms/store/user/$1",
+                "chain": "first"
+              },
+              { "lfn": "/+store/(.*)",
+                "pfn": "/cms/store/$1"
+              }
+            ]
+          },
+          {
+            "protocol": "root",
+            "rules": [
+              { "lfn": "/+(.*)",
+                "pfn": "root://host.domain//pnfs/$1",
+                "chain": "second"
+              }
+            ]
+          }
       ],
       "type": "DISK",
       "rse": "T1_US_FNAL_Disk",