Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove eostools dependency from cmsIO #184

Merged
merged 2 commits into from
Jun 20, 2017
Merged

Remove eostools dependency from cmsIO #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c4c94c2
eostools: avoid cmsIO module that is no longer available in the stand…
gpetruc Jun 13, 2017
4ca40d5
samples_13TeV_80X_susySignalsPriv: Remove missing private Higgsino sa…
gpetruc Jun 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 31 additions & 114 deletions Production/python/eostools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,38 @@
import sys
import os
import re
import pprint
import shutil
import io
import zlib

def setCAFPath():
"""Hack to get the CAF scripts on the PYTHONPATH"""
caf = '/afs/cern.ch/cms/caf/python'

if caf not in sys.path:
sys.path.append(caf)
setCAFPath()
try:
import cmsIO
except ImportError as e:
import logging
logger = logging.getLogger(__name__)
logger.warning(str(e))
logger.warning("wasn't able to import cmsIO, which this job might not need unless it uses EOS.")
import subprocess

def splitPFN(pfn):
"""Split the PFN in to { <protocol>, <host>, <path>, <opaque> }"""
groups = re.match("^(\w\+)://([^/]+)/(/[^?]+)(\?.*)?", pfn)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I missed one problem here, the backslash before the first + should not be there. Instead, it should be:
groups = re.match("^(\w+)://([^/]+)/(/[^?]+)(\?.*)?", pfn)
which for instance works for root://eoscms.cern.ch//eos/cms/store/cmst3/user/jngadiub/ntuple returning
root
eoscms.cern.ch
/eos/cms/store/cmst3/user/jngadiub/ntuple

if not groups: raise RuntimeError, "Malformed pfn: '%s'" % pfn
return (groups.group(1), groups.group(2), groups.group(3), groups.group(4))

def _runCommand(cmd):
myCommand = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE )
( out, err ) = myCommand.communicate()
if myCommand.returncode != 0:
print >> sys.stderr, "Command (%s) failed with return code: %d" % ( cmd, myCommand.returncode )
print >> sys.stderr, err
return out,err,myCommand.returncode

def runXRDCommand(path, cmd, *args):
"""Run an xrd command.

!!! Will, what is happening in case of problem?
??? At some point, should return a list of lines instead of a string."""

lfn = eosToLFN(path)
#print "lfn:", lfn, cmd
tokens = cmsIO.splitPFN(lfnToPFN(lfn))
tokens = splitPFN(path)

command = ['xrd', tokens[1], cmd, tokens[2]]
command.extend(args)
runner = cmsIO.cmsFileManip()
# print ' '.join(command)
return runner.runCommand(command)
return _runCommand(command)

def runEOSCommand(path, cmd, *args):
"""Run an eos command.
Expand All @@ -48,16 +45,13 @@ def runEOSCommand(path, cmd, *args):
I think we should really try and raise an exception in case of problems.
should be possible as the return code is provided in the tuple returned by runner."""

lfn = eosToLFN(path)
pfn = lfnToPFN(lfn)
tokens = cmsIO.splitPFN(pfn)
tokens = splitPFN(path)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is another issue here (or in general), it calls splitPFN with a /eos/... path and not with the protocol prepended, so the regex evaluation above will fail.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the error is later on at lines 194-196. It works when replacing those lines with:

pfn = lfnToPFN(path)
if not isEOSFile(pfn):
    runEOSCommand(pfn,'mkdir','-p')


#obviously, this is not nice
command = ['/afs/cern.ch/project/eos/installation/pro/bin/eos.select', cmd]
command.extend(args)
command.append(tokens[2])
runner = cmsIO.cmsFileManip()
return runner.runCommand(command)
return _runCommand(command)

def isLFN( path ):
"""Tests whether this path is a CMS LFN (name starts with /store...)"""
Expand Down Expand Up @@ -93,20 +87,10 @@ def lfnToPFN( path, tfcProt = 'rfio'):

if path.startswith("/store/"):
path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
elif path.startswith("/pnfs/psi.ch/cms/trivcat/"):
path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
#print "path to cmsFile():", path
entity = cmsIO.cmsFile( path, tfcProt )
# tokens = cmsIO.splitPFN(entity.pfn)
pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)

pfn = entity.pfn
if tfcProt == 'rfio' and \
entity.path.startswith("/eos/cms/") and \
str(entity.stat()).startswith("Error 3011: Unable to stat"):

pfn.replace("/eos/cms","/castor/cern.ch/cms")
pfn.replace("eoscms","castorcms")
if ":" in path: pfn = path
else: pfn = "file:"+path
return pfn


Expand All @@ -133,66 +117,15 @@ def isEOSDir( path ):
root://eoscms.cern.ch//eos/cms/

Otherwise, returns False.

WARNING!! This function does not check for path existence,
and returns true also for plain files.
!!! Will, is my summary correct?
"""
if os.path.exists( path ):
# path does not exist
# COLIN: I think this condition could be removed,
# as it duplicates the following one.
return False
if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'):
# neither an EOS PFN or a LFN.
return False
# at this stage, we must have an EOS PFN or an LFN
pfn = lfnToPFN(eosToLFN(path))
tokens = cmsIO.splitPFN(pfn)
return tokens and tokens[1].lower().startswith('eos')
return path.startswith('/eos') or path.startswith('/store') or path.startswith('root://eoscms.cern.ch//eos/cms/') or path.startswith('root://eoscms//eos/cms/')

#also define an alias for backwards compatibility
isCastorDir = isEOSDir


def isEOSFile( path, tfcProt = 'rfio'):
"""Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory

False
"""
def isEOSFile( path ):
"""Returns True if path is a file or directory stored on EOS (checks for path existence)"""
if not isEOSDir(path): return False
_, _, ret = runEOSCommand( path, 'ls')
return ret == 0

Expand Down Expand Up @@ -261,9 +194,6 @@ def createEOSDir( path ):
if not isEOSFile(lfn):
# if not isDirectory(lfn):
runEOSCommand(lfn,'mkdir','-p')
# entity = cmsIO.cmsFile( lfn,"stageout")
# entity.mkdir([])
# # print 'created ', path
if isDirectory(path):
return path
else:
Expand Down Expand Up @@ -340,6 +270,8 @@ def listFiles(path, rec = False, full_info = False):
result.extend(allFiles)
return result
# -- listing on EOS --
if not isEOSDir(path):
raise RuntimeError, "Bad path '%s': not existent, and not in EOS" % path
cmd = 'dirlist'
if rec:
cmd = 'dirlistrec'
Expand All @@ -357,19 +289,6 @@ def listFiles(path, rec = False, full_info = False):
result.append( tokens[4] )
return result

def which(cmd):
command = ['which', cmd]
runner = cmsIO.cmsFileManip()
out, _, _ = runner.runCommand(command)

lines = [line for line in out.split('\n') if line]
if len(lines) == 1:
return lines[0]
elif len(lines) == 2:
return lines[1]
else:
return lines

def ls(path, rec = False):
"""Provides a simple list of the specified directory, works on EOS and locally"""
return [eosToLFN(t) for t in listFiles(path, rec)]
Expand Down Expand Up @@ -479,7 +398,7 @@ def xrdcp(src, dest):
dest = eosToLFN(dest)
pfn_dest = lfnToPFN(dest)
if isDirectory(dest):
tokens = cmsIO.splitPFN(pfn_dest)
tokens = splitPFN(pfn_dest)
pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2])
elif os.path.exists(dest):
pfn_dest = dest
Expand Down Expand Up @@ -525,8 +444,7 @@ def _xrdcpSingleFile( pfn_src, pfn_dest):
# print ' '.join(command)
run = True
if run:
runner = cmsIO.cmsFileManip()
out, err, ret = runner.runCommand(command)
out, err, ret = _runCommand(command)
if err:
print >> sys.stderr, out
print >> sys.stderr, err
Expand Down Expand Up @@ -573,5 +491,4 @@ def cmsStage( absDestDir, files, force):
command.append(eosToLFN(fname))
command.append(eosToLFN(absDestDir))
print ' '.join(command)
runner = cmsIO.cmsFileManip()
runner.runCommand(command)
_runCommand(command)
10 changes: 5 additions & 5 deletions RootTools/python/samples/samples_13TeV_80X_susySignalsPriv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,11 @@

##Higgsino
## ------------------------------------------------------
Higgsino_300_120=kreator.makeMCComponentFromLocal("Higgsino_300_120","/higgsino_300_120_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
Higgsino_300_80=kreator.makeMCComponentFromLocal("Higgsino_300_80","/higgsino_300_80_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
Higgsino_500_120=kreator.makeMCComponentFromLocal("Higgsino_500_120","higgsino_500_120_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
Higgsino_500_80=kreator.makeMCComponentFromLocal("Higgsino_500_80","higgsino_500_80_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
Higgsino = [Higgsino_300_120, Higgsino_300_80, Higgsino_500_120, Higgsino_500_80]
#Higgsino_300_120=kreator.makeMCComponentFromLocal("Higgsino_300_120","/higgsino_300_120_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
#Higgsino_300_80=kreator.makeMCComponentFromLocal("Higgsino_300_80","/higgsino_300_80_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
#Higgsino_500_120=kreator.makeMCComponentFromLocal("Higgsino_500_120","higgsino_500_120_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
#Higgsino_500_80=kreator.makeMCComponentFromLocal("Higgsino_500_80","higgsino_500_80_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root")
Higgsino = [] #Higgsino_300_120, Higgsino_300_80, Higgsino_500_120, Higgsino_500_80]



Expand Down