diff --git a/Production/python/eostools.py b/Production/python/eostools.py index d50ede15299c..f075326bcdb1 100644 --- a/Production/python/eostools.py +++ b/Production/python/eostools.py @@ -5,25 +5,24 @@ import sys import os import re -import pprint import shutil import io import zlib - -def setCAFPath(): - """Hack to get the CAF scripts on the PYTHONPATH""" - caf = '/afs/cern.ch/cms/caf/python' - - if caf not in sys.path: - sys.path.append(caf) -setCAFPath() -try: - import cmsIO -except ImportError as e: - import logging - logger = logging.getLogger(__name__) - logger.warning(str(e)) - logger.warning("wasn't able to import cmsIO, which this job might not need unless it uses EOS.") +import subprocess + +def splitPFN(pfn): + """Split the PFN in to { , , , }""" + groups = re.match("^(\w\+)://([^/]+)/(/[^?]+)(\?.*)?", pfn) + if not groups: raise RuntimeError, "Malformed pfn: '%s'" % pfn + return (groups.group(1), groups.group(2), groups.group(3), groups.group(4)) + +def _runCommand(cmd): + myCommand = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + ( out, err ) = myCommand.communicate() + if myCommand.returncode != 0: + print >> sys.stderr, "Command (%s) failed with return code: %d" % ( cmd, myCommand.returncode ) + print >> sys.stderr, err + return out,err,myCommand.returncode def runXRDCommand(path, cmd, *args): """Run an xrd command. @@ -31,15 +30,13 @@ def runXRDCommand(path, cmd, *args): !!! Will, what is happening in case of problem? ??? At some point, should return a list of lines instead of a string.""" - lfn = eosToLFN(path) #print "lfn:", lfn, cmd - tokens = cmsIO.splitPFN(lfnToPFN(lfn)) + tokens = splitPFN(path) command = ['xrd', tokens[1], cmd, tokens[2]] command.extend(args) - runner = cmsIO.cmsFileManip() # print ' '.join(command) - return runner.runCommand(command) + return _runCommand(command) def runEOSCommand(path, cmd, *args): """Run an eos command. @@ -48,16 +45,13 @@ def runEOSCommand(path, cmd, *args): I think we should really try and raise an exception in case of problems. should be possible as the return code is provided in the tuple returned by runner.""" - lfn = eosToLFN(path) - pfn = lfnToPFN(lfn) - tokens = cmsIO.splitPFN(pfn) + tokens = splitPFN(path) #obviously, this is not nice command = ['/afs/cern.ch/project/eos/installation/pro/bin/eos.select', cmd] command.extend(args) command.append(tokens[2]) - runner = cmsIO.cmsFileManip() - return runner.runCommand(command) + return _runCommand(command) def isLFN( path ): """Tests whether this path is a CMS LFN (name starts with /store...)""" @@ -93,20 +87,10 @@ def lfnToPFN( path, tfcProt = 'rfio'): if path.startswith("/store/"): path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/") - if path.startswith("/pnfs/psi.ch/cms/trivcat/"): + elif path.startswith("/pnfs/psi.ch/cms/trivcat/"): path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//") - #print "path to cmsFile():", path - entity = cmsIO.cmsFile( path, tfcProt ) -# tokens = cmsIO.splitPFN(entity.pfn) - pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path) - - pfn = entity.pfn - if tfcProt == 'rfio' and \ - entity.path.startswith("/eos/cms/") and \ - str(entity.stat()).startswith("Error 3011: Unable to stat"): - - pfn.replace("/eos/cms","/castor/cern.ch/cms") - pfn.replace("eoscms","castorcms") + if ":" in path: pfn = path + else: pfn = "file:"+path return pfn @@ -133,66 +117,15 @@ def isEOSDir( path ): root://eoscms.cern.ch//eos/cms/ Otherwise, returns False. - - WARNING!! This function does not check for path existence, - and returns true also for plain files. - !!! Will, is my summary correct? """ - if os.path.exists( path ): - # path does not exist - # COLIN: I think this condition could be removed, - # as it duplicates the following one. - return False - if not path.startswith('/eos') and not path.startswith('/store') and not path.startswith('root://eoscms.cern.ch//eos/cms/'): - # neither an EOS PFN or a LFN. - return False - # at this stage, we must have an EOS PFN or an LFN - pfn = lfnToPFN(eosToLFN(path)) - tokens = cmsIO.splitPFN(pfn) - return tokens and tokens[1].lower().startswith('eos') + return path.startswith('/eos') or path.startswith('/store') or path.startswith('root://eoscms.cern.ch//eos/cms/') or path.startswith('root://eoscms//eos/cms/') #also define an alias for backwards compatibility isCastorDir = isEOSDir - -def isEOSFile( path, tfcProt = 'rfio'): - """Returns True if path is a file or directory stored on EOS (checks for path existence) - ??? This function does not behave well if passed a non EOS path... - returns lots of error messages like: ->>> eostools.isEOSFile('/store/asdfasfd') -Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2 -ls: s: No such file or directory -ls: t: No such file or directory -ls: o: No such file or directory -ls: r: No such file or directory -ls: e: No such file or directory -ls: a: No such file or directory -ls: s: No such file or directory -ls: d: No such file or directory -ls: f: No such file or directory -ls: a: No such file or directory -ls: s: No such file or directory -ls: f: No such file or directory -ls: d: No such file or directory -ls: /store: No such file or directory - -ls: s: No such file or directory -ls: t: No such file or directory -ls: o: No such file or directory -ls: r: No such file or directory -ls: e: No such file or directory -ls: a: No such file or directory -ls: s: No such file or directory -ls: d: No such file or directory -ls: f: No such file or directory -ls: a: No such file or directory -ls: s: No such file or directory -ls: f: No such file or directory -ls: d: No such file or directory -ls: /store: No such file or directory - -False - """ +def isEOSFile( path ): + """Returns True if path is a file or directory stored on EOS (checks for path existence)""" + if not isEOSDir(path): return False _, _, ret = runEOSCommand( path, 'ls') return ret == 0 @@ -261,9 +194,6 @@ def createEOSDir( path ): if not isEOSFile(lfn): # if not isDirectory(lfn): runEOSCommand(lfn,'mkdir','-p') - # entity = cmsIO.cmsFile( lfn,"stageout") - # entity.mkdir([]) - # # print 'created ', path if isDirectory(path): return path else: @@ -340,6 +270,8 @@ def listFiles(path, rec = False, full_info = False): result.extend(allFiles) return result # -- listing on EOS -- + if not isEOSDir(path): + raise RuntimeError, "Bad path '%s': not existent, and not in EOS" % path cmd = 'dirlist' if rec: cmd = 'dirlistrec' @@ -357,19 +289,6 @@ def listFiles(path, rec = False, full_info = False): result.append( tokens[4] ) return result -def which(cmd): - command = ['which', cmd] - runner = cmsIO.cmsFileManip() - out, _, _ = runner.runCommand(command) - - lines = [line for line in out.split('\n') if line] - if len(lines) == 1: - return lines[0] - elif len(lines) == 2: - return lines[1] - else: - return lines - def ls(path, rec = False): """Provides a simple list of the specified directory, works on EOS and locally""" return [eosToLFN(t) for t in listFiles(path, rec)] @@ -479,7 +398,7 @@ def xrdcp(src, dest): dest = eosToLFN(dest) pfn_dest = lfnToPFN(dest) if isDirectory(dest): - tokens = cmsIO.splitPFN(pfn_dest) + tokens = splitPFN(pfn_dest) pfn_dest = '%s://%s//%s/' % (tokens[0],tokens[1],tokens[2]) elif os.path.exists(dest): pfn_dest = dest @@ -525,8 +444,7 @@ def _xrdcpSingleFile( pfn_src, pfn_dest): # print ' '.join(command) run = True if run: - runner = cmsIO.cmsFileManip() - out, err, ret = runner.runCommand(command) + out, err, ret = _runCommand(command) if err: print >> sys.stderr, out print >> sys.stderr, err @@ -573,5 +491,4 @@ def cmsStage( absDestDir, files, force): command.append(eosToLFN(fname)) command.append(eosToLFN(absDestDir)) print ' '.join(command) - runner = cmsIO.cmsFileManip() - runner.runCommand(command) + _runCommand(command) diff --git a/RootTools/python/samples/samples_13TeV_80X_susySignalsPriv.py b/RootTools/python/samples/samples_13TeV_80X_susySignalsPriv.py index 17aba0e9800c..c357d9288b6a 100644 --- a/RootTools/python/samples/samples_13TeV_80X_susySignalsPriv.py +++ b/RootTools/python/samples/samples_13TeV_80X_susySignalsPriv.py @@ -144,11 +144,11 @@ ##Higgsino ## ------------------------------------------------------ -Higgsino_300_120=kreator.makeMCComponentFromLocal("Higgsino_300_120","/higgsino_300_120_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") -Higgsino_300_80=kreator.makeMCComponentFromLocal("Higgsino_300_80","/higgsino_300_80_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") -Higgsino_500_120=kreator.makeMCComponentFromLocal("Higgsino_500_120","higgsino_500_120_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") -Higgsino_500_80=kreator.makeMCComponentFromLocal("Higgsino_500_80","higgsino_500_80_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") -Higgsino = [Higgsino_300_120, Higgsino_300_80, Higgsino_500_120, Higgsino_500_80] +#Higgsino_300_120=kreator.makeMCComponentFromLocal("Higgsino_300_120","/higgsino_300_120_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") +#Higgsino_300_80=kreator.makeMCComponentFromLocal("Higgsino_300_80","/higgsino_300_80_v5/","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") +#Higgsino_500_120=kreator.makeMCComponentFromLocal("Higgsino_500_120","higgsino_500_120_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") +#Higgsino_500_80=kreator.makeMCComponentFromLocal("Higgsino_500_80","higgsino_500_80_v5","/afs/cern.ch/work/b/bschneid/public/higgsino-points/%s",".*root") +Higgsino = [] #Higgsino_300_120, Higgsino_300_80, Higgsino_500_120, Higgsino_500_80]