Skip to content

Commit

Permalink
unit tests for the new stage out implementation (storage.json)
Browse files Browse the repository at this point in the history
  • Loading branch information
nhduongvn committed Jan 3, 2024
1 parent 7ebfb24 commit 4268fd3
Show file tree
Hide file tree
Showing 5 changed files with 173 additions and 6 deletions.
13 changes: 7 additions & 6 deletions test/python/WMCore_t/Misc_t/Runtime_t.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ def miniStartup(thisDir=os.getcwd()):
Or run this in subprocess
"""

Bootstrap.setupLogging(thisDir)
job = Bootstrap.loadJobDefinition()
task = Bootstrap.loadTask(job)
Expand Down Expand Up @@ -151,6 +150,7 @@ def setupTestWorkload(self, workloadName='Test', emulator=True):
if not os.path.exists(siteConfigPath):
os.makedirs(siteConfigPath)
shutil.copy(os.path.join(self.thisDirPath, 'site-local-config.xml'), siteConfigPath)
shutil.copy(os.path.join(self.thisDirPath, 'storage.json'), os.path.join(siteConfigPath,'..'))
environment = rereco.data.section_('environment')
environment.CMS_PATH = workloadDir
environment.SITECONFIG_PATH = os.path.join(workloadDir, 'SITECONF/local')
Expand Down Expand Up @@ -304,7 +304,7 @@ def runJobs(self, workload):
for primeTask in workload.taskIterator():
listOfTasks.append(primeTask)
# Only run primeTasks for now

for task in listOfTasks:
jobName = task.name()
taskDir = os.path.join(self.unpackDir, jobName, 'job')
Expand All @@ -314,8 +314,9 @@ def runJobs(self, workload):
# Scream, run around in panic, blow up machine
print("About to run jobs")
print(taskDir)
#SITECONFIG_PATH is not available here so set it up so that site config can be loaded in Bootstrap.createInitialReport inside miniStartup
os.environ['SITECONFIG_PATH'] = os.path.realpath(os.path.join(taskDir,'../../../basicWorkload/SITECONF/local'))
miniStartup(thisDir=taskDir)

# When exiting, go back to where you started
os.chdir(self.initialDir)
sys.path.remove(taskDir)
Expand Down Expand Up @@ -409,7 +410,7 @@ def testB_EmulatorTest(self):
self.createWMBSComponents(workload=workload)

self.unpackComponents(workload=workload)

self.runJobs(workload=workload)

# Check the report
Expand All @@ -419,8 +420,8 @@ def testB_EmulatorTest(self):
cmsReport = report.data.cmsRun1

# Now validate the report
self.assertEqual(report.getSiteName(), {})
# self.assertEqual(report.data.hostName, socket.gethostname())
self.assertEqual(report.getSiteName(), 'T1_US_FNAL')
#self.assertEqual(report.data.hostName, socket.gethostname())
self.assertTrue(report.data.completed)

# Should have status 0 (emulator job)
Expand Down
5 changes: 5 additions & 0 deletions test/python/WMCore_t/Misc_t/site-local-config.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
<command value="test-copy"/>
<catalog url="trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap"/>
</local-stage-out>
<stage-out>
<method volume="FNAL_dCache_EOS" protocol="XRootD" command="xrdcp" option="-p"/>
<method volume="FNAL_dCache_EOS" protocol="SRMv2"/>
<method volume="FNAL_dCache_EOS" protocol="WebDAV"/>
</stage-out>
<calib-data>
<frontier-connect>
<load balance="proxies"/>
Expand Down
74 changes: 74 additions & 0 deletions test/python/WMCore_t/Misc_t/storage.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
[
{ "site": "T1_US_FNAL",
"volume": "FNAL_dCache_EOS",
"protocols": [
{ "protocol": "xrootd",
"access": "global-ro",
"comment": "xrootd read via site redirector",
"prefix": "root://cmsxrootd-site.fnal.gov/"
},
{ "protocol": "XRootD",
"access": "global-rw",
"comment": "xrootd write to dCache/EOS endpoint directly",
"rules": [
{ "lfn": "/+store/temp/user/(.*)",
"pfn": "root://cmseos.fnal.gov//eos/uscms/store/temp/user/$1"
},
{ "lfn": "/+store/(.*)",
"pfn": "root://cmsdcadisk.fnal.gov//dcache/uscmsdisk/store/$1"
}
]
},
{ "protocol": "SRMv2",
"access": "global-rw",
"rules": [
{ "lfn": "/+store/temp/user/(.*)",
"pfn": "gsiftp://cmseos-gridftp.fnal.gov//eos/uscms/store/temp/user/$1"
},
{ "lfn": "/+store/(.*)",
"pfn": "srm://cmsdcadisk.fnal.gov:8443/srm/managerv2?SFN=/dcache/uscmsdisk/store/$1"
}
]
},
{ "protocol": "WebDAV",
"comment": "Ignoring CRAB stageout to EOS -- all disk dCache",
"access": "global-rw",
"prefix": "davs://cmsdcadisk.fnal.gov:2880/dcache/uscmsdisk"
}
],
"type": "DISK",
"rse": "T1_US_FNAL_Disk",
"fts": [ "https://cmsfts3.fnal.gov:8446", "https://fts3-cms.cern.ch:8446" ],
"loadtest": true
},

{ "site": "T1_US_FNAL",
"volume": "FNAL_MSS",
"protocols": [
{ "protocol": "SRMv2",
"access": "global-rw",
"prefix": "srm://cmsdcatape.fnal.gov:8443/srm/managerv2?SFN=/11"
},
{ "protocol": "WebDAV",
"access": "global-rw",
"prefix": "davs://cmsdcatape.fnal.gov:2880/WAX/11"
}
],
"type": "TAPE",
"rse": "T1_US_FNAL_Tape",
"fts": [ "https://cmsfts3.fnal.gov:8446", "https://fts3-cms.cern.ch:8446" ]
},

{ "site": "T1_US_FNAL",
"volume": "American_Federation",
"protocols": [
{ "protocol": "XRootD",
"access": "global-ro",
"prefix": "root://cmsxrootd.fnal.gov/"
}
],
"type": "DISK",
"rse": null,
"fts": []
}
]
8 changes: 8 additions & 0 deletions test/python/WMCore_t/Storage_t/StageOutMgr_t.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ def setUp(self):
os.putenv('SITECONFIG_PATH', os.getcwd())

def testStageOutMgr(self):
configFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t/T1_DE_KIT/JobConfig",
"site-local-config.xml")
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = configFileName
os.environ['SITECONFIG_PATH'] = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_DE_KIT")
Expand Down Expand Up @@ -61,6 +65,10 @@ def testStageOutMgr(self):
stageOutMgr.cleanSuccessfulStageOuts()

#test subsite
configFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t/T1_DE_KIT/KIT-T3/JobConfig",
"site-local-config.xml")
os.environ["WMAGENT_SITE_CONFIG_OVERRIDE"] = configFileName
os.environ['SITECONFIG_PATH'] = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_DE_KIT/KIT-T3")
Expand Down
79 changes: 79 additions & 0 deletions test/python/WMCore_t/Storage_t/T2_DE_DESY/storage.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
[
{ "site": "T2_DE_DESY",
"volume": "DESY_dCache",
"protocols": [
{ "protocol": "pnfs",
"access": "virtual",
"rules": [
{ "lfn": "/+store/unmerged/(.*)",
"pfn": "/pnfs/desy.de/cms/tier2/unmerged/$1"
},
{ "lfn": "/+store/temp/(.*)",
"pfn": "/pnfs/desy.de/cms/tier2/temp/$1"
},
{ "lfn": "/+(.*)",
"pfn": "/pnfs/desy.de/cms/tier2/$1"
}
]
},
{ "protocol": "gsidcap",
"access": "site-ro",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "gsidcap://dcache-cms-gsidcap.desy.de:22128/$1",
"chain": "pnfs"
}
]
},
{ "protocol": "dcap",
"access": "site-ro",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "dcap://dcache-cms-dcap.desy.de/$1",
"chain": "pnfs"
}
]
},
{ "protocol": "SRMv2",
"access": "global-rw",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=/$1",
"chain": "pnfs"
}
]
},
{ "protocol": "XRootD",
"access": "global-rw",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "root://dcache-cms-xrootd.desy.de:1094//$1"
}
]
},
{ "protocol": "WebDAV",
"access": "global-rw",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "davs://dcache-cms-webdav-wan.desy.de:2880/$1",
"chain": "pnfs"
}
]
},
{ "protocol": "davs-lan",
"access": "global-rw",
"rules": [
{ "lfn": "/+(.*)",
"pfn": "davs://dcache-cms-webdav-job.desy.de:2880/$1",
"chain": "pnfs"
}
]
}
],
"type": "DISK",
"rse": "T2_DE_DESY",
"fts": [ "https://fts3-cms.cern.ch:8446", "https://lcgfts3.gridpp.rl.ac.uk:8446" ],
"loadtest": true
}
]

0 comments on commit 4268fd3

Please sign in to comment.