Added option --make-light-tarball to use symbolic links for data fold…

…er (#1184)
cms-analysis · Dec 16, 2019 · d47da03 · d47da03
1 parent 534af5e
commit d47da03
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 6 deletions.
diff --git a/MetaData/python/jobs_utils.py b/MetaData/python/jobs_utils.py
@@ -93,7 +93,8 @@ def __init__(self,
                             help="default: %default"),
                 make_option("-m","--max-resubmissions",dest="maxResub", type="int",default=2),
                 make_option("-N","--ncpu",dest="ncpu", type="int",default=cpu_count()),
-                make_option("--nCondorCpu",dest="ncondorcpu", type="int",default=1),
+                make_option("--nCondorCpu",dest="ncondorcpu", type="int",default=1,help="Number of cpu cores per job to request from condor"),
+                make_option("--make-light-tarball",dest="lighttarball", action="store_true",default=False,help="Include datafolders only as symbolic link in tarball"),
                 make_option("-H","--hadd",dest="hadd",default=False, action="store_true",
                             help="hadd output files when all jobs are finished."
                             ),
@@ -269,8 +270,8 @@ def firstRun(self):
         if options.useTarball:
             apset = os.path.abspath(pset)
             self.jobFactory.mkTarball("%s/sandbox.tgz" % os.path.abspath(options.outputDir),
-                                      tarball_entries=[apset,"python","lib","bin","src/flashgg/MetaData/python"],tarball_patterns=[("src/*","data"), ("external/*","data"), ("src/*","toolbox")],
-                                      tarball_transform="'s,%s,pset.py,'" % (apset.lstrip("/"))
+                                      tarball_entries=[apset,"python","lib","bin","src/flashgg/MetaData/python"],tarball_patterns=[("src/*","data"), ("external/*","data"), ("src/*","toolbox"), ("src/*","python")],
+                                      tarball_transform="'s,%s,pset.py,'" % (apset.lstrip("/")), light=self.options.lighttarball
                                       )
             if not options.queue:
                 print "\nWARNING: You specified the --use-tarball option but no batch queue. The tarball was created but the jobs won't actually use it."

diff --git a/MetaData/python/parallel.py b/MetaData/python/parallel.py
@@ -195,7 +195,7 @@ def __call__(self,cmd):
             script += "export SCRAM_ARCH=%s\n" % os.environ['SCRAM_ARCH']
             script += "scram project CMSSW %s\n" % os.environ['CMSSW_VERSION']
             script += "cd %s\n" % os.environ['CMSSW_VERSION']
-            script += "tar zxf %s\n" % self.tarball
+            script += "tar zxf %s -h\n" % self.tarball
             script += "cp src/XGBoostCMSSW/XGBoostInterface/toolbox/*xml config/toolbox/$SCRAM_ARCH/tools/selected/\n"
             script += "scram setup rabit\n"
             script += "scram setup xgboost\n"
@@ -299,7 +299,7 @@ def setTarball(self,tarball):
     # ------------------------------------------------------------------------------------------------
     def mkTarball(self,tarball=None,
                   tarball_entries=["python","lib","bin","external","flashgg/MetaData/python/PU_MixFiles_2017_miniaodv2_310"],tarball_patterns=[("src/*","data")],
-                  tarball_transform=None):
+                  tarball_transform=None, light=False):
 
         self.tarball = tarball
         content=tarball_entries
@@ -317,7 +317,11 @@ def mkTarball(self,tarball=None,
         args = []
         if tarball_transform:
             args.extend( ["--transform",tarball_transform] )
-        args.extend(["-h","--show-transformed","-zvcf",tarball])
+
+        if light:
+            args.extend(["--show-transformed","-zvcf",tarball]) #add -h to follow symlinks and include stuff from there
+        else:
+            args.extend(["-h","--show-transformed","-zvcf",tarball])
         args.extend(content)
         print 
         print "Preparing tarball with the following content:"