diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/alignment.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/alignment.py deleted file mode 100644 index 8520c1024a123..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/alignment.py +++ /dev/null @@ -1,299 +0,0 @@ -from __future__ import absolute_import -import collections -import os -import re - -from . import configTemplates -from .helperFunctions import parsecolor, parsestyle, replaceByMap, clean_name, getTagsMap -from .TkAlExceptions import AllInOneError - -class Alignment(object): - condShorts = { - "TrackerAlignmentErrorExtendedRcd": { - "zeroAPE_phase0": { - "connectString":("frontier://FrontierProd" - "/CMS_CONDITIONS"), - "tagName": "TrackerIdealGeometryErrorsExtended210_mc", - "labelName": "" - }, - "zeroAPE_phase1": { - "connectString":("frontier://FrontierProd" - "/CMS_CONDITIONS"), - "tagName": "TrackerAlignmentErrorsExtended_Upgrade2017_design_v0", - "labelName": "" - }, - }, - "TrackerSurfaceDeformationRcd": { - "zeroDeformations": { - "connectString":("frontier://FrontierProd" - "/CMS_CONDITIONS"), - "tagName": "TrackerSurfaceDeformations_zero", - "labelName": "" - }, - }, - } - def __init__(self, name, config, runGeomComp = "1"): - section = "alignment:%s"%name - if not config.has_section( section ): - raise AllInOneError("section %s not found. Please define the " - "alignment!"%section) - config.checkInput(section, - knownSimpleOptions = ['globaltag', 'style', 'color', 'title', 'mp', 'mp_alignments', 'mp_deformations', 'mp_APEs', 'hp', 'hp_alignments', 'hp_deformations', 'sm', 'sm_alignments', 'sm_deformations'], - knownKeywords = ['condition']) - self.name = clean_name(name) - if config.exists(section,"title"): - self.title = config.get(section,"title") - else: - self.title = self.name - if (int(runGeomComp) != 1): - self.name += "_run" + runGeomComp - self.title += " run " + runGeomComp - if "|" in self.title or "," in self.title or '"' in self.title: - msg = "The characters '|', '\"', and ',' cannot be used in the alignment title!" - raise AllInOneError(msg) - self.runGeomComp = runGeomComp - self.globaltag = config.get( section, "globaltag" ) - self.conditions = self.__getConditions( config, section ) - - self.color = config.get(section,"color") - self.style = config.get(section,"style") - - self.color = str(parsecolor(self.color)) - self.style = str(parsestyle(self.style)) - - def __shorthandExists(self, theRcdName, theShorthand): - """Method which checks, if `theShorthand` is a valid shorthand for the - given `theRcdName`. - - Arguments: - - `theRcdName`: String which specifies the database record. - - `theShorthand`: String which specifies the shorthand to check. - """ - - if (theRcdName in self.condShorts) and \ - (theShorthand in self.condShorts[theRcdName]): - return True - else: - return False - - def __getConditions( self, theConfig, theSection ): - conditions = [] - for option in theConfig.options( theSection ): - if option in ("mp", "mp_alignments", "mp_deformations", "mp_APEs", "hp", "hp_alignments", "hp_deformations", "sm", "sm_alignments", "sm_deformations"): - matches = [re.match(_, option) for _ in ("^(..)$", "^(..)_alignments$", "^(..)_deformations$", "^(..)_APEs$")] - assert sum(bool(_) for _ in matches) == 1, option - condPars = theConfig.get(theSection, option).split(",") - condPars = [_.strip() for _ in condPars] - if matches[0]: - alignments = True - deformations = True - APEs = {"hp": False, "mp": True}[option] - elif matches[1]: - alignments = True - deformations = False - APEs = False - option = matches[1].group(1) - elif matches[2]: - alignments = False - deformations = True - APEs = False - option = matches[2].group(1) - elif matches[3]: - alignments = False - deformations = False - APEs = True - option = matches[3].group(1) - else: - assert False - - if option == "mp": - if len(condPars) == 1: - number, = condPars - jobm = None - elif len(condPars) == 2: - number, jobm = condPars - else: - raise AllInOneError("Up to 2 arguments accepted for {} (job number, and optionally jobm index)".format(option)) - - folder = "/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/{}{}/".format(option, number) - if not os.path.exists(folder): - raise AllInOneError(folder+" does not exist.") - folder = os.path.join(folder, "jobData") - jobmfolders = set() - if jobm is None: - for filename in os.listdir(folder): - if re.match("jobm([0-9]*)", filename) and os.path.isdir(os.path.join(folder, filename)): - jobmfolders.add(filename) - if len(jobmfolders) == 0: - raise AllInOneError("No jobm or jobm(number) folder in {}".format(folder)) - elif len(jobmfolders) == 1: - folder = os.path.join(folder, jobmfolders.pop()) - else: - raise AllInOneError( - "Multiple jobm or jobm(number) folders in {}\n".format(folder) - + ", ".join(jobmfolders) + "\n" - + "Please specify 0 for jobm, or a number for one of the others." - ) - elif jobm == "0": - folder = os.path.join(folder, "jobm") - if os.path.exists(folder + "0"): - raise AllInOneError("Not set up to handle a folder named jobm0") - else: - folder = os.path.join(folder, "jobm{}".format(jobm)) - - dbfile = os.path.join(folder, "alignments_MP.db") - if not os.path.exists(dbfile): - raise AllInOneError("No file {}. Maybe your alignment folder is corrupted, or maybe you specified the wrong jobm?".format(dbfile)) - - elif option in ("hp", "sm"): - if len(condPars) == 1: - number, = condPars - iteration = None - elif len(condPars) == 2: - number, iteration = condPars - else: - raise AllInOneError("Up to 2 arguments accepted for {} (job number, and optionally iteration)".format(option)) - folder = "/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/{}{}".format(option, number) - if not os.path.exists(folder): - raise AllInOneError(folder+" does not exist.") - if iteration is None: - for filename in os.listdir(folder): - match = re.match("alignments_iter([0-9]*).db", filename) - if match: - if iteration is None or int(match.group(1)) > iteration: - iteration = int(match.group(1)) - if iteration is None: - raise AllInOneError("No alignments in {}".format(folder)) - dbfile = os.path.join(folder, "alignments_iter{}.db".format(iteration)) - if not os.path.exists(dbfile): - raise AllInOneError("No file {}.".format(dbfile)) - - if "Deformations" not in getTagsMap(dbfile).keys(): - deformations = False #so that hp = XXXX works whether or not deformations were aligned - if not alignments: #then it's specified with hp_deformations, which is a mistake - raise AllInOneError("{}{} has no deformations".format(option, number)) - - else: - assert False, option - - if alignments: - conditions.append({"rcdName": "TrackerAlignmentRcd", - "connectString": "sqlite_file:"+dbfile, - "tagName": "Alignments", - "labelName": ""}) - if deformations: - conditions.append({"rcdName": "TrackerSurfaceDeformationRcd", - "connectString": "sqlite_file:"+dbfile, - "tagName": "Deformations", - "labelName": ""}) - if APEs: - conditions.append({"rcdName": "TrackerAlignmentErrorExtendedRcd", - "connectString": "sqlite_file:"+dbfile, - "tagName": "AlignmentErrorsExtended", - "labelName": ""}) - - elif option.startswith( "condition " ): - rcdName = option.split( "condition " )[1] - condPars = theConfig.get( theSection, option ).split( "," ) - if len(condPars) == 1: - if len(condPars[0])==0: - msg = ("In section [%s]: '%s' is used with too few " - "arguments. A connect_string and a tag are " - "required!"%(theSection, option)) - raise AllInOneError(msg) - elif self.__shorthandExists(rcdName, condPars[0]): - shorthand = condPars[0] - condPars = [ - self.condShorts[rcdName][shorthand]["connectString"], - self.condShorts[rcdName][shorthand]["tagName"], - self.condShorts[rcdName][shorthand]["labelName"]] - elif rcdName == "TrackerAlignmentErrorExtendedRcd" and condPars[0] == "zeroAPE": - raise AllInOneError("Please specify either zeroAPE_phase0 or zeroAPE_phase1") - #can probably make zeroAPE an alias of zeroAPE_phase1 at some point, - #but not sure if now is the time - else: - msg = ("In section [%s]: '%s' is used with '%s', " - "which is an unknown shorthand for '%s'. Either " - "provide at least a connect_string and a tag or " - "use a known shorthand.\n" - %(theSection, option, condPars[0], rcdName)) - if rcdName in self.condShorts: - msg += "Known shorthands for '%s':\n"%(rcdName) - theShorts = self.condShorts[rcdName] - knownShorts = [("\t"+key+": " - +theShorts[key]["connectString"]+"," - +theShorts[key]["tagName"]+"," - +theShorts[key]["labelName"]) \ - for key in theShorts] - msg+="\n".join(knownShorts) - else: - msg += ("There are no known shorthands for '%s'." - %(rcdName)) - raise AllInOneError(msg) - if len( condPars ) == 2: - condPars.append( "" ) - if len(condPars) > 3: - msg = ("In section [%s]: '%s' is used with too many " - "arguments. A maximum of 3 arguments is allowed." - %(theSection, option)) - raise AllInOneError(msg) - conditions.append({"rcdName": rcdName.strip(), - "connectString": condPars[0].strip(), - "tagName": condPars[1].strip(), - "labelName": condPars[2].strip()}) - - rcdnames = collections.Counter(condition["rcdName"] for condition in conditions) - if rcdnames and max(rcdnames.values()) >= 2: - raise AllInOneError("Some conditions are specified multiple times (possibly through mp or hp options)!\n" - + ", ".join(rcdname for rcdname, count in rcdnames.items() if count >= 2)) - - for condition in conditions: - self.__testDbExist(condition["connectString"], condition["tagName"]) - - return conditions - - def __testDbExist(self, dbpath, tagname): - if dbpath.startswith("sqlite_file:"): - if not os.path.exists( dbpath.split("sqlite_file:")[1] ): - raise AllInOneError("could not find file: '%s'"%dbpath.split("sqlite_file:")[1]) - elif tagname not in getTagsMap(dbpath).values(): - raise AllInOneError("{} does not exist in {}".format(tagname, dbpath)) - - def restrictTo( self, restriction ): - result = [] - if not restriction == None: - for mode in self.mode: - if mode in restriction: - result.append( mode ) - self.mode = result - - def getRepMap( self ): - result = { - "name": self.name, - "title": self.title, - "color": self.color, - "style": self.style, - "runGeomComp": self.runGeomComp, - "GlobalTag": self.globaltag - } - return result - - def getConditions(self): - """This function creates the configuration snippet to override - global tag conditions. - """ - if len( self.conditions ): - loadCond = ("\nimport CalibTracker.Configuration." - "Common.PoolDBESSource_cfi\n") - for cond in self.conditions: - if not cond["labelName"] == "": - temp = configTemplates.conditionsTemplate.replace( - "tag = cms.string('.oO[tagName]Oo.')", - ("tag = cms.string('.oO[tagName]Oo.')," - "\nlabel = cms.untracked.string('.oO[labelName]Oo.')")) - else: - temp = configTemplates.conditionsTemplate - loadCond += replaceByMap( temp, cond ) - else: - loadCond = "" - return loadCond diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/alternateValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/alternateValidationTemplates.py deleted file mode 100644 index fd0763e6909e5..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/alternateValidationTemplates.py +++ /dev/null @@ -1,5 +0,0 @@ -###################################################################### -###################################################################### -otherTemplate = """ -schum schum -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/configTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/configTemplates.py deleted file mode 100644 index 31c75cd743cb5..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/configTemplates.py +++ /dev/null @@ -1,400 +0,0 @@ -from __future__ import absolute_import -from .alternateValidationTemplates import * -from .offlineValidationTemplates import * -from .primaryVertexValidationTemplates import * -from .primaryVertexResolutionTemplates import * -from .geometryComparisonTemplates import * -from .monteCarloValidationTemplates import * -from .trackSplittingValidationTemplates import * -from .zMuMuValidationTemplates import * -from .TkAlExceptions import AllInOneError -from .overlapValidationTemplates import * - -###################################################################### -###################################################################### -### ### -### General Templates ### -### ### -###################################################################### -###################################################################### - -###################################################################### -###################################################################### -loadGlobalTagTemplate=""" -#Global tag -process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff") -from Configuration.AlCa.GlobalTag import GlobalTag -process.GlobalTag = GlobalTag(process.GlobalTag,".oO[GlobalTag]Oo.") -""" - - -###################################################################### -###################################################################### -conditionsTemplate=""" -process.conditionsIn.oO[rcdName]Oo. = CalibTracker.Configuration.Common.PoolDBESSource_cfi.poolDBESSource.clone( - connect = cms.string('.oO[connectString]Oo.'), - toGet = cms.VPSet(cms.PSet(record = cms.string('.oO[rcdName]Oo.'), - tag = cms.string('.oO[tagName]Oo.') - ) - ) - ) -process.prefer_conditionsIn.oO[rcdName]Oo. = cms.ESPrefer("PoolDBESSource", "conditionsIn.oO[rcdName]Oo.") -""" - - -###################################################################### -###################################################################### -#batch job execution -scriptTemplate="""#!/bin/bash -#init -#ulimit -v 3072000 -#export STAGE_SVCCLASS=cmscafuser -#save path to the condor batch working directory (/pool/condor) - -export CONDORWORKDIR=`pwd -P` -echo CONDOR working directory is $CONDORWORKDIR -source /afs/cern.ch/cms/caf/setup.sh -export X509_USER_PROXY=.oO[scriptsdir]Oo./.user_proxy -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scramv1 ru -sh` -#mkdir -p .oO[datadir]Oo. &>! /dev/null - -#remove possible result file from previous runs -previous_results=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo.) -for file in ${previous_results} -do - if [ ${file} = /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[outputFile]Oo. ] - then - xrdcp -f root://eoscms//eos/cms${file} root://eoscms//eos/cms${file}.bak - fi -done - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - rm -f .oO[workdir]Oo./* - cd .oO[workdir]Oo. -else - mkdir -p $CONDORWORKDIR/TkAllInOneTool - cd $CONDORWORKDIR/TkAllInOneTool -fi - -# rm -f .oO[workdir]Oo./* -# cd .oO[workdir]Oo. - -#run -pwd -df -h . -which cmsRun -.oO[CommandLine]Oo. -echo "----" -echo "List of files in $(pwd):" -ls -ltr -echo "----" -echo "" - - -#retrieve -mkdir -p .oO[logdir]Oo. >&! /dev/null -gzip -f LOGFILE_*_.oO[name]Oo..log -find . -maxdepth 1 -name "LOGFILE*.oO[alignmentName]Oo.*" -print | xargs -I {} bash -c "cp {} .oO[logdir]Oo." - -#copy root files to eos -mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. -if [ .oO[parallelJobs]Oo. -eq 1 ] -then - root_files=$(ls --color=never -d *.oO[alignmentName]Oo.*.root) -else - root_files=$(ls --color=never -d *.oO[alignmentName]Oo._.oO[nIndex]Oo.*.root) -fi -echo ${root_files} - -for file in ${root_files} -do - xrdcp -f ${file} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. - echo ${file} -done - -#cleanup -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - rm -rf .oO[workdir]Oo. -fi -echo "done." -""" - - -###################################################################### -###################################################################### -cfgTemplate=""" -import FWCore.ParameterSet.Config as cms - -process = cms.Process(".oO[ProcessName]Oo.") - -.oO[datasetDefinition]Oo. -.oO[Bookkeeping]Oo. -.oO[LoadBasicModules]Oo. -.oO[TrackSelectionRefitting]Oo. -.oO[LoadGlobalTagTemplate]Oo. -.oO[condLoad]Oo. -.oO[ValidationConfig]Oo. -.oO[FileOutputTemplate]Oo. - -.oO[DefinePath]Oo. - -print("Done") -""" - - -###################################################################### -###################################################################### -Bookkeeping = """ -process.options = cms.untracked.PSet( - wantSummary = cms.untracked.bool(False), - Rethrow = cms.untracked.vstring("ProductNotFound"), # make this exception fatal - fileMode = cms.untracked.string('NOMERGE') # no ordering needed, but calls endRun/beginRun etc. at file boundaries -) - -process.load("FWCore.MessageLogger.MessageLogger_cfi") -process.MessageLogger.cerr.FwkReport.reportEvery = 1000 -process.MessageLogger.cout.enableStatistics = cms.untracked.bool(True) -""" - - -###################################################################### -###################################################################### -CommonTrackSelectionRefitting = """ -import Alignment.CommonAlignment.tools.trackselectionRefitting as trackselRefit -process.seqTrackselRefit = trackselRefit.getSequence(process, '.oO[trackcollection]Oo.', - isPVValidation=.oO[ispvvalidation]Oo., - TTRHBuilder='.oO[ttrhbuilder]Oo.', - usePixelQualityFlag=.oO[usepixelqualityflag]Oo., - openMassWindow=.oO[openmasswindow]Oo., - cosmicsDecoMode=.oO[cosmicsdecomode]Oo., - cosmicsZeroTesla=.oO[cosmics0T]Oo., - momentumConstraint=.oO[momentumconstraint]Oo., - cosmicTrackSplitting=.oO[istracksplitting]Oo., - use_d0cut=.oO[use_d0cut]Oo., - ) - -.oO[trackhitfiltercommands]Oo. -""" - - -###################################################################### -###################################################################### -SingleTrackRefitter = """ -process.load("RecoTracker.TrackProducer.TrackRefitters_cff") -process.TrackRefitter.src = ".oO[TrackCollection]Oo." -process.TrackRefitter.TTRHBuilder = ".oO[ttrhbuilder]Oo." -process.TrackRefitter.NavigationSchool = "" -""" - - -###################################################################### -###################################################################### -LoadBasicModules = """ -process.load("RecoVertex.BeamSpotProducer.BeamSpot_cff") -process.load("Configuration.Geometry.GeometryDB_cff") -process.load('Configuration.StandardSequences.Services_cff') -process.load("Configuration.StandardSequences..oO[magneticField]Oo._cff") -""" - - -###################################################################### -###################################################################### -FileOutputTemplate = """ -process.TFileService = cms.Service("TFileService", - fileName = cms.string('.oO[outputFile]Oo.') -) -""" - - -###################################################################### -###################################################################### -DefinePath_CommonSelectionRefitting = """ -process.p = cms.Path( -process.seqTrackselRefit*.oO[ValidationSequence]Oo.) -""" - -###################################################################### -###################################################################### -mergeTemplate="""#!/bin/bash -CWD=`pwd -P` -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scramv1 ru -sh` - - -.oO[createResultsDirectory]Oo. - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - cd .oO[workdir]Oo. -else - cd $CWD -fi -echo "Working directory: $(pwd -P)" - -############################################################################### -# download root files from eos -root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \ - | grep ".root$" | grep -v "result.root$") -#for file in ${root_files} -#do -# xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} . -# echo ${file} -#done - - -#run -.oO[DownloadData]Oo. -.oO[CompareAlignments]Oo. - -.oO[RunValidationPlots]Oo. - -# clean-up -# ls -l *.root -rm -f *.root - -#zip stdout and stderr from the farm jobs -cd .oO[logdir]Oo. -find . -name "*.stderr" -exec gzip -f {} \; -find . -name "*.stdout" -exec gzip -f {} \; -""" - - - -###################################################################### -###################################################################### -mergeParallelOfflineTemplate="""#!/bin/bash -CWD=`pwd -P` -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scramv1 ru -sh` - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - cd .oO[workdir]Oo. -else - cd $CWD -fi -echo "Working directory: $(pwd -P)" - -############################################################################### -# download root files from eos -root_files=$(ls /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. \ - | grep ".root$" | grep -v "result.root$") -#for file in ${root_files} -#do -# xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${file} . -# echo ${file} -#done - - -#run -.oO[DownloadData]Oo. -""" - -###################################################################### -###################################################################### -createResultsDirectoryTemplate=""" -#create results-directory and copy used configuration there -mkdir -p .oO[datadir]Oo. -cp .oO[logdir]Oo./usedConfiguration.ini .oO[datadir]Oo. -""" - - -###################################################################### -###################################################################### -mergeParallelResults=""" - -.oO[beforeMerge]Oo. -.oO[doMerge]Oo. - -# create log file -ls -al .oO[mergeParallelFilePrefixes]Oo. > .oO[datadir]Oo./log_rootfilelist.txt - -# Remove parallel job files -.oO[rmUnmerged]Oo. -""" - - -###################################################################### -###################################################################### -compareAlignmentsExecution=""" -#merge for .oO[validationId]Oo. if it does not exist or is not up-to-date -echo -e "\n\nComparing validations" -mkdir -p /eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./ -cp .oO[Alignment/OfflineValidation]Oo./scripts/compareFileAges.C . -root -x -q -b -l "compareFileAges.C(\\\"root://eoscms.cern.ch//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root\\\", \\\".oO[compareStringsPlain]Oo.\\\")" -comparisonNeeded=${?} - -if [[ ${comparisonNeeded} -eq 1 ]] -then - cp .oO[compareAlignmentsPath]Oo. . - root -x -q -b -l '.oO[compareAlignmentsName]Oo.++(\".oO[compareStrings]Oo.\", ".oO[legendheader]Oo.", ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo.", .oO[bigtext]Oo.)' - mv result.root .oO[validationId]Oo._result.root - xrdcp -f .oO[validationId]Oo._result.root root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo. -else - echo ".oO[validationId]Oo._result.root is up-to-date, no need to compare again." - xrdcp -f root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root . -fi -""" - - -###################################################################### -###################################################################### -crabCfgTemplate=""" -[CRAB] -jobtype = cmssw -scheduler = caf -use_server = 0 - -[CMSSW] -datasetpath = .oO[dataset]Oo. -pset = .oO[cfgFile]Oo. -total_number_of_.oO[McOrData]Oo. -number_of_jobs = .oO[numberOfJobs]Oo. -output_file = .oO[outputFile]Oo. -runselection = .oO[runRange]Oo. -lumi_mask = .oO[JSON]Oo. - -[USER] -return_data = 0 -copy_data = 1 -storage_element = T2_CH_CERN -user_remote_dir = .oO[eosdir]Oo. -ui_working_dir = .oO[crabWorkingDir]Oo. -# script_exe = .oO[script]Oo. -# .oO[email]Oo. - -[CAF] -queue = .oO[queue]Oo. -""" - - - - -###################################################################### -###################################################################### -### ### -### Alternate Templates ### -### ### -###################################################################### -###################################################################### - - -def alternateTemplate( templateName, alternateTemplateName ): - - if not templateName in globals().keys(): - msg = "unknown template to replace %s"%templateName - raise AllInOneError(msg) - if not alternateTemplateName in globals().keys(): - msg = "unknown template to replace %s"%alternateTemplateName - raise AllInOneError(msg) - globals()[ templateName ] = globals()[ alternateTemplateName ] - # = eval("configTemplates.%s"%"alternateTemplate") diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/crabWrapper.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/crabWrapper.py deleted file mode 100644 index 4500f61a4cbdf..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/crabWrapper.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import -import sys -import os -import subprocess -from .TkAlExceptions import AllInOneError - -# script which needs to be sourced for use of crab -crabSourceScript = '/afs/cern.ch/cms/ccs/wm/scripts/Crab/crab.sh' - -# source the environment variables needed for crab -sourceStr = ( 'cd $CMSSW_BASE/src;' - 'source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.sh;' - 'eval `scramv1 runtime -sh`;' - 'source ' + crabSourceScript + ' && env' ) -sourceCmd = ['bash', '-c', sourceStr ] -sourceProc = subprocess.Popen(sourceCmd, stdout = subprocess.PIPE) -for line in sourceProc.stdout: - (key, _, value) = line.partition("=") - os.environ[key] = value.replace("\n","") -sourceProc.communicate() - -# source variables from crab wrapper script -crabFile = open('/'.join([os.environ["CRABPYTHON"],'crab'])) -theLines = crabFile.readlines() -theLine = [] -for line in theLines: - if ( line[0] == '#' ) or \ - ( line == ' python $CRABPYTHON/crab.py $*\n' ): - continue - theLine.append( line ) -tempFilePath = "tempCrab" -tempFile = open( tempFilePath, "w" ) -tempFile.write( ''.join(theLine) ) -tempFile.close() -crabStr = ('source tempCrab && env' ) -crabCmd = ['bash', '-c', crabStr ] -crabProc = subprocess.Popen(crabCmd, stdout = subprocess.PIPE) -for line in crabProc.stdout: - (key, _, value) = line.partition("=") - os.environ[key] = value.replace("\n","") -crabProc.communicate() -os.remove( tempFilePath ) - -# add sourced paths to search path of python -sys.path.extend( os.environ["PYTHONPATH"].split( ':' ) ) - -import crab -import crab_exceptions - -class CrabWrapper(object): - def run( self, options ): - theCrab = crab.Crab() - try: - theCrab.initialize_( options ) - theCrab.run() - except crab_exceptions.CrabException as e: - raise AllInOneError( str( e ) ) - del theCrab - - -if __name__ == "__main__": - theCrab = CrabWrapper() - theCrabOptions = {"-create":"", - "-cfg":"TkAlOfflineValidation.shiftPlots.crab.cfg"} - theCrab.run( theCrabOptions ) - - theCrabOptions = {"-submit":""} - theCrab.run( theCrabOptions ) - - theCrabOptions = {"-status":""} - theCrab.run( theCrabOptions ) - - theCrabOptions = {"-getoutput":""} - try: - theCrab.run( theCrabOptions ) - except AllInOneError as e: - print("crab: ", e) diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/dataset.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/dataset.py deleted file mode 100644 index 7c3fd08b497b3..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/dataset.py +++ /dev/null @@ -1,941 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import -# idea stolen from: -# http://cmssw.cvs.cern.ch/cgi-bin/cmssw.cgi/CMSSW/ -# PhysicsTools/PatAlgos/python/tools/cmsswVersionTools.py -from builtins import range -import bisect -import datetime -import json -import os -import re -import sys - -import Utilities.General.cmssw_das_client as das_client -from FWCore.PythonUtilities.LumiList import LumiList - -from .helperFunctions import cache -from .TkAlExceptions import AllInOneError - -class Dataset(object): - def __init__( self, datasetName, dasLimit = 0, tryPredefinedFirst = True, - cmssw = os.environ["CMSSW_BASE"], cmsswrelease = os.environ["CMSSW_RELEASE_BASE"], - magneticfield = None, dasinstance = None): - self.__name = datasetName - self.__origName = datasetName - self.__dasLimit = dasLimit - self.__dasinstance = dasinstance - self.__cmssw = cmssw - self.__cmsswrelease = cmsswrelease - self.__firstusedrun = None - self.__lastusedrun = None - self.__parentDataset = None - - # check, if dataset name matches CMS dataset naming scheme - if re.match( r'/.+/.+/.+', self.__name ): - self.__official = True - fileName = "Dataset" + self.__name.replace("/","_") + "_cff.py" - else: - self.__official = False - fileName = self.__name + "_cff.py" - - searchPath1 = os.path.join( self.__cmssw, "python", - "Alignment", "OfflineValidation", - fileName ) - searchPath2 = os.path.join( self.__cmssw, "src", - "Alignment", "OfflineValidation", - "python", fileName ) - searchPath3 = os.path.join( self.__cmsswrelease, - "python", "Alignment", - "OfflineValidation", fileName ) - if self.__official and not tryPredefinedFirst: - self.__predefined = False - elif os.path.exists( searchPath1 ): - self.__predefined = True - self.__filename = searchPath1 - elif os.path.exists( searchPath2 ): - msg = ("The predefined dataset '%s' does exist in '%s', but " - "you need to run 'scram b' first." - %( self.__name, searchPath2 )) - if self.__official: - print(msg) - print("Getting the data from DAS again. To go faster next time, run scram b.") - else: - raise AllInOneError( msg ) - elif os.path.exists( searchPath3 ): - self.__predefined = True - self.__filename = searchPath3 - elif self.__official: - self.__predefined = False - else: - msg = ("The predefined dataset '%s' does not exist. Please " - "create it first or check for typos."%( self.__name )) - raise AllInOneError( msg ) - - if self.__predefined and self.__official: - self.__name = "Dataset" + self.__name.replace("/","_") - - if magneticfield is not None: - try: - magneticfield = float(magneticfield) - except ValueError: - raise AllInOneError("Bad magneticfield {} which can't be converted to float".format(magneticfield)) - self.__inputMagneticField = magneticfield - - self.__dataType = self.__getDataType() - self.__magneticField = self.__getMagneticField() - - - def __chunks( self, theList, n ): - """ Yield successive n-sized chunks from theList. - """ - for i in range( 0, len( theList ), n ): - yield theList[i:i+n] - - __source_template= ("%(header)s" - "%(importCms)s" - "import FWCore.PythonUtilities.LumiList as LumiList\n\n" - "%(goodLumiSecStr)s" - "readFiles = cms.untracked.vstring()\n" - "secFiles = cms.untracked.vstring()\n" - "%(process)ssource = cms.Source(\"PoolSource\",\n" - "%(lumiStr)s" - "%(tab)s secondaryFileNames =" - "secFiles,\n" - "%(tab)s fileNames = readFiles\n" - ")\n" - "%(files)s\n" - "%(lumiSecExtend)s\n" - "%(process)smaxEvents = cms.untracked.PSet( " - "input = cms.untracked.int32(int(%(nEvents)s)) )\n" - "%(skipEventsString)s\n") - - __dummy_source_template = ("readFiles = cms.untracked.vstring()\n" - "secFiles = cms.untracked.vstring()\n" - "%(process)ssource = cms.Source(\"PoolSource\",\n" - "%(tab)s secondaryFileNames =" - "secFiles,\n" - "%(tab)s fileNames = readFiles\n" - ")\n" - "readFiles.extend(['dummy_File.root'])\n" - "%(process)smaxEvents = cms.untracked.PSet( " - "input = cms.untracked.int32(int(%(nEvents)s)) )\n" - "%(skipEventsString)s\n") - - def __lumiSelectionSnippet( self, jsonPath = None, firstRun = None, lastRun = None ): - lumiSecExtend = "" - if firstRun or lastRun or jsonPath: - if not jsonPath: - selectedRunList = self.__getRunList() - if firstRun: - selectedRunList = [ run for run in selectedRunList \ - if self.__findInJson(run, "run_number") >= firstRun ] - if lastRun: - selectedRunList = [ run for run in selectedRunList \ - if self.__findInJson(run, "run_number") <= lastRun ] - lumiList = [ str( self.__findInJson(run, "run_number") ) + ":1-" \ - + str( self.__findInJson(run, "run_number") ) + ":max" \ - for run in selectedRunList ] - splitLumiList = list( self.__chunks( lumiList, 255 ) ) - else: - theLumiList = None - try: - theLumiList = LumiList ( filename = jsonPath ) - except ValueError: - pass - - if theLumiList is not None: - allRuns = theLumiList.getRuns() - runsToRemove = [] - for run in allRuns: - if firstRun and int( run ) < firstRun: - runsToRemove.append( run ) - if lastRun and int( run ) > lastRun: - runsToRemove.append( run ) - theLumiList.removeRuns( runsToRemove ) - splitLumiList = list( self.__chunks( - theLumiList.getCMSSWString().split(','), 255 ) ) - if not (splitLumiList and splitLumiList[0] and splitLumiList[0][0]): - splitLumiList = None - else: - with open(jsonPath) as f: - jsoncontents = f.read() - if "process.source.lumisToProcess" in jsoncontents: - msg = "%s is not a json file, but it seems to be a CMSSW lumi selection cff snippet. Trying to use it" % jsonPath - if firstRun or lastRun: - msg += ("\n (after applying firstRun and/or lastRun)") - msg += ".\nPlease note that, depending on the format of this file, it may not work as expected." - msg += "\nCheck your config file to make sure that it worked properly." - print(msg) - - runlist = self.__getRunList() - if firstRun or lastRun: - self.__firstusedrun = -1 - self.__lastusedrun = -1 - jsoncontents = re.sub(r"\d+:(\d+|max)(-\d+:(\d+|max))?", self.getForceRunRangeFunction(firstRun, lastRun), jsoncontents) - jsoncontents = (jsoncontents.replace("'',\n","").replace("''\n","") - .replace('"",\n','').replace('""\n','')) - self.__firstusedrun = max(self.__firstusedrun, int(self.__findInJson(runlist[0],"run_number"))) - self.__lastusedrun = min(self.__lastusedrun, int(self.__findInJson(runlist[-1],"run_number"))) - if self.__lastusedrun < self.__firstusedrun: - jsoncontents = None - else: - self.__firstusedrun = int(self.__findInJson(runlist[0],"run_number")) - self.__lastusedrun = int(self.__findInJson(runlist[-1],"run_number")) - lumiSecExtend = jsoncontents - splitLumiList = None - else: - raise AllInOneError("%s is not a valid json file!" % jsonPath) - - if splitLumiList and splitLumiList[0] and splitLumiList[0][0]: - lumiSecStr = [ "',\n'".join( lumis ) \ - for lumis in splitLumiList ] - lumiSecStr = [ "lumiSecs.extend( [\n'" + lumis + "'\n] )" \ - for lumis in lumiSecStr ] - lumiSecExtend = "\n".join( lumiSecStr ) - runlist = self.__getRunList() - self.__firstusedrun = max(int(splitLumiList[0][0].split(":")[0]), int(self.__findInJson(runlist[0],"run_number"))) - self.__lastusedrun = min(int(splitLumiList[-1][-1].split(":")[0]), int(self.__findInJson(runlist[-1],"run_number"))) - elif lumiSecExtend: - pass - else: - msg = "You are trying to run a validation without any runs! Check that:" - if firstRun or lastRun: - msg += "\n - firstRun/begin and lastRun/end are correct for this dataset, and there are runs in between containing data" - if jsonPath: - msg += "\n - your JSON file is correct for this dataset, and the runs contain data" - if (firstRun or lastRun) and jsonPath: - msg += "\n - firstRun/begin and lastRun/end are consistent with your JSON file" - raise AllInOneError(msg) - - else: - if self.__inputMagneticField is not None: - pass #never need self.__firstusedrun or self.__lastusedrun - else: - runlist = self.__getRunList() - self.__firstusedrun = int(self.__findInJson(self.__getRunList()[0],"run_number")) - self.__lastusedrun = int(self.__findInJson(self.__getRunList()[-1],"run_number")) - - return lumiSecExtend - - def __fileListSnippet(self, crab=False, parent=False, firstRun=None, lastRun=None, forcerunselection=False): - if crab: - files = "" - else: - splitFileList = list( self.__chunks( self.fileList(firstRun=firstRun, lastRun=lastRun, forcerunselection=forcerunselection), 255 ) ) - if not splitFileList: - raise AllInOneError("No files found for dataset {}. Check the spelling, or maybe specify another das instance?".format(self.__name)) - fileStr = [ "',\n'".join( files ) for files in splitFileList ] - fileStr = [ "readFiles.extend( [\n'" + files + "'\n] )" \ - for files in fileStr ] - files = "\n".join( fileStr ) - - if parent: - splitParentFileList = list( self.__chunks( self.fileList(parent=True, firstRun=firstRun, lastRun=lastRun, forcerunselection=forcerunselection), 255 ) ) - parentFileStr = [ "',\n'".join( parentFiles ) for parentFiles in splitParentFileList ] - parentFileStr = [ "secFiles.extend( [\n'" + parentFiles + "'\n] )" \ - for parentFiles in parentFileStr ] - parentFiles = "\n".join( parentFileStr ) - files += "\n\n" + parentFiles - - return files - - def __createSnippet( self, jsonPath = None, begin = None, end = None, - firstRun = None, lastRun = None, repMap = None, - crab = False, parent = False ): - - if firstRun: - firstRun = int( firstRun ) - if lastRun: - lastRun = int( lastRun ) - if ( begin and firstRun ) or ( end and lastRun ): - msg = ( "The Usage of " - + "'begin' & 'firstRun' " * int( bool( begin and - firstRun ) ) - + "and " * int( bool( ( begin and firstRun ) and - ( end and lastRun ) ) ) - + "'end' & 'lastRun' " * int( bool( end and lastRun ) ) - + "is ambigous." ) - raise AllInOneError( msg ) - if begin or end: - ( firstRun, lastRun ) = self.convertTimeToRun( - begin = begin, end = end, firstRun = firstRun, - lastRun = lastRun ) - if ( firstRun and lastRun ) and ( firstRun > lastRun ): - msg = ( "The lower time/runrange limit ('begin'/'firstRun') " - "chosen is greater than the upper time/runrange limit " - "('end'/'lastRun').") - raise AllInOneError( msg ) - - lumiSecExtend = self.__lumiSelectionSnippet(jsonPath=jsonPath, firstRun=firstRun, lastRun=lastRun) - lumiStr = goodLumiSecStr = "" - if lumiSecExtend: - goodLumiSecStr = "lumiSecs = cms.untracked.VLuminosityBlockRange()\n" - lumiStr = " lumisToProcess = lumiSecs,\n" - - files = self.__fileListSnippet(crab=crab, parent=parent, firstRun=firstRun, lastRun=lastRun, forcerunselection=False) - - theMap = repMap - theMap["files"] = files - theMap["json"] = jsonPath - theMap["lumiStr"] = lumiStr - theMap["goodLumiSecStr"] = goodLumiSecStr%( theMap ) - theMap["lumiSecExtend"] = lumiSecExtend - if crab: - dataset_snippet = self.__dummy_source_template%( theMap ) - else: - dataset_snippet = self.__source_template%( theMap ) - return dataset_snippet - - def __find_lt( self, a, x ): - 'Find rightmost value less than x' - i = bisect.bisect_left( a, x ) - if i: - return i-1 - raise ValueError - - def __find_ge( self, a, x): - 'Find leftmost item greater than or equal to x' - i = bisect.bisect_left( a, x ) - if i != len( a ): - return i - raise ValueError - - def __findInJson(self, jsondict, strings): - if isinstance(strings, str): - strings = [ strings ] - - if len(strings) == 0: - return jsondict - if isinstance(jsondict,dict): - if strings[0] in jsondict: - try: - return self.__findInJson(jsondict[strings[0]], strings[1:]) - except KeyError: - pass - else: - for a in jsondict: - if strings[0] in a: - try: - return self.__findInJson(a[strings[0]], strings[1:]) - except (TypeError, KeyError): #TypeError because a could be a string and contain strings[0] - pass - #if it's not found - raise KeyError("Can't find " + strings[0]) - - def forcerunrange(self, firstRun, lastRun, s): - """s must be in the format run1:lum1-run2:lum2""" - s = s.group() - run1 = s.split("-")[0].split(":")[0] - lum1 = s.split("-")[0].split(":")[1] - try: - run2 = s.split("-")[1].split(":")[0] - lum2 = s.split("-")[1].split(":")[1] - except IndexError: - run2 = run1 - lum2 = lum1 - if int(run2) < firstRun or int(run1) > lastRun: - return "" - if int(run1) < firstRun or firstRun < 0: - run1 = firstRun - lum1 = 1 - if int(run2) > lastRun: - run2 = lastRun - lum2 = "max" - if int(run1) < self.__firstusedrun or self.__firstusedrun < 0: - self.__firstusedrun = int(run1) - if int(run2) > self.__lastusedrun: - self.__lastusedrun = int(run2) - return "%s:%s-%s:%s" % (run1, lum1, run2, lum2) - - def getForceRunRangeFunction(self, firstRun, lastRun): - def forcerunrangefunction(s): - return self.forcerunrange(firstRun, lastRun, s) - return forcerunrangefunction - - def __getData( self, dasQuery, dasLimit = 0 ): - dasData = das_client.get_data(dasQuery, dasLimit) - if isinstance(dasData, str): - jsondict = json.loads( dasData ) - else: - jsondict = dasData - # Check, if the DAS query fails - try: - error = self.__findInJson(jsondict,["data","error"]) - except KeyError: - error = None - if error or self.__findInJson(jsondict,"status") != 'ok' or "data" not in jsondict: - try: - jsonstr = self.__findInJson(jsondict,"reason") - except KeyError: - jsonstr = str(jsondict) - if len(jsonstr) > 10000: - jsonfile = "das_query_output_%i.txt" - i = 0 - while os.path.lexists(jsonfile % i): - i += 1 - jsonfile = jsonfile % i - theFile = open( jsonfile, "w" ) - theFile.write( jsonstr ) - theFile.close() - msg = "The DAS query returned an error. The output is very long, and has been stored in:\n" + jsonfile - else: - msg = "The DAS query returned a error. Here is the output\n" + jsonstr - msg += "\nIt's possible that this was a server error. If so, it may work if you try again later" - raise AllInOneError(msg) - return self.__findInJson(jsondict,"data") - - def __getDataType( self ): - if self.__predefined: - with open(self.__filename) as f: - datatype = None - for line in f.readlines(): - if line.startswith("#data type: "): - if datatype is not None: - raise AllInOneError(self.__filename + " has multiple 'data type' lines.") - datatype = line.replace("#data type: ", "").replace("\n","") - return datatype - return "unknown" - - dasQuery_type = ( 'dataset dataset=%s instance=%s detail=true | grep dataset.datatype,' - 'dataset.name'%( self.__name, self.__dasinstance ) ) - data = self.__getData( dasQuery_type ) - - try: - return self.__findInJson(data, ["dataset", "datatype"]) - except KeyError: - print ("Cannot find the datatype of the dataset '%s'\n" - "It may not be possible to automatically find the magnetic field,\n" - "and you will not be able run in CRAB mode" - %( self.name() )) - return "unknown" - - def __getParentDataset( self ): - dasQuery = "parent dataset=" + self.__name + " instance="+self.__dasinstance - data = self.__getData( dasQuery ) - try: - return self.__findInJson(data, ["parent", "name"]) - except KeyError: - raise AllInOneError("Cannot find the parent of the dataset '" + self.__name + "'\n" - "Here is the DAS output:\n" + str(jsondict) + - "\nIt's possible that this was a server error. If so, it may work if you try again later") - - def __getMagneticField( self ): - Bfieldlocation = os.path.join( self.__cmssw, "python", "Configuration", "StandardSequences" ) - if not os.path.isdir(Bfieldlocation): - Bfieldlocation = os.path.join( self.__cmsswrelease, "python", "Configuration", "StandardSequences" ) - Bfieldlist = [ f.replace("_cff.py",'') \ - for f in os.listdir(Bfieldlocation) \ - if f.startswith("MagneticField_") and f.endswith("_cff.py") ] - Bfieldlist.sort( key = lambda Bfield: -len(Bfield) ) #Put it in order of decreasing length, so that searching in the name gives the longer match - - if self.__inputMagneticField is not None: - if self.__inputMagneticField == 3.8: - return "MagneticField" - elif self.__inputMagneticField == 0: - return "MagneticField_0T" - else: - raise ValueError("Unknown input magnetic field {}".format(self.__inputMagneticField)) - - if self.__predefined: - with open(self.__filename) as f: - datatype = None - Bfield = None - for line in f.readlines(): - if line.startswith("#data type: "): - if datatype is not None: - raise AllInOneError(self.__filename + " has multiple 'data type' lines.") - datatype = line.replace("#data type: ", "").replace("\n","") - datatype = datatype.split("#")[0].strip() - if line.startswith("#magnetic field: "): - if Bfield is not None: - raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.") - Bfield = line.replace("#magnetic field: ", "").replace("\n","") - Bfield = Bfield.split("#")[0].strip() - if Bfield is not None: - Bfield = Bfield.split(",")[0] - if Bfield in Bfieldlist or Bfield == "unknown": - return Bfield - else: - print("Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield) - print("Using Bfield='unknown' - this will revert to the default") - return "unknown" - elif datatype == "data": - return "MagneticField" #this should be in the "#magnetic field" line, but for safety in case it got messed up - else: - return "unknown" - - if self.__dataType == "data": - return "MagneticField" - - #try to find the magnetic field from DAS - #it seems to be there for the newer (7X) MC samples, except cosmics - dasQuery_B = ('dataset dataset=%s instance=%s'%(self.__name, self.__dasinstance)) - data = self.__getData( dasQuery_B ) - - try: - Bfield = self.__findInJson(data, ["dataset", "mcm", "sequences", "magField"]) - if Bfield in Bfieldlist: - return Bfield - elif Bfield == "38T" or Bfield == "38T_PostLS1": - return "MagneticField" - elif "MagneticField_" + Bfield in Bfieldlist: - return "MagneticField_" + Bfield - elif Bfield == "": - pass - else: - print("Your dataset has magnetic field '%s', which does not exist in your CMSSW version!" % Bfield) - print("Using Bfield='unknown' - this will revert to the default magnetic field") - return "unknown" - except KeyError: - pass - - for possibleB in Bfieldlist: - if (possibleB != "MagneticField" - and possibleB.replace("MagneticField_","") in self.__name.replace("TkAlCosmics0T", "")): - #final attempt - try to identify the dataset from the name - #all cosmics dataset names contain "TkAlCosmics0T" - if possibleB == "MagneticField_38T" or possibleB == "MagneticField_38T_PostLS1": - return "MagneticField" - return possibleB - - return "unknown" - - def __getMagneticFieldForRun( self, run = -1, tolerance = 0.5 ): - """For MC, this returns the same as the previous function. - For data, it gets the magnetic field from the runs. This is important for - deciding which template to use for offlinevalidation - """ - if self.__dataType == "mc" and self.__magneticField == "MagneticField": - return 3.8 #For 3.8T MC the default MagneticField is used - if self.__inputMagneticField is not None: - return self.__inputMagneticField - if "T" in self.__magneticField: - Bfield = self.__magneticField.split("T")[0].replace("MagneticField_","") - try: - return float(Bfield) / 10.0 #e.g. 38T and 38T_PostLS1 both return 3.8 - except ValueError: - pass - if self.__predefined: - with open(self.__filename) as f: - Bfield = None - for line in f.readlines(): - if line.startswith("#magnetic field: ") and "," in line: - if Bfield is not None: - raise AllInOneError(self.__filename + " has multiple 'magnetic field' lines.") - return float(line.replace("#magnetic field: ", "").split(",")[1].split("#")[0].strip()) - - if run > 0: - dasQuery = ('run=%s instance=%s detail=true'%(run, self.__dasinstance)) #for data - data = self.__getData(dasQuery) - try: - return self.__findInJson(data, ["run","bfield"]) - except KeyError: - return "unknown Can't get the magnetic field for run %s from DAS" % run - - #run < 0 - find B field for the first and last runs, and make sure they're compatible - # (to within tolerance) - #NOT FOOLPROOF! The magnetic field might go up and then down, or vice versa - if self.__firstusedrun is None or self.__lastusedrun is None: - return "unknown Can't get the exact magnetic field for the dataset until data has been retrieved from DAS." - firstrunB = self.__getMagneticFieldForRun(self.__firstusedrun) - lastrunB = self.__getMagneticFieldForRun(self.__lastusedrun) - try: - if abs(firstrunB - lastrunB) <= tolerance: - return .5*(firstrunB + lastrunB) - print(firstrunB, lastrunB, tolerance) - return ("unknown The beginning and end of your run range for %s\n" - "have different magnetic fields (%s, %s)!\n" - "Try limiting the run range using firstRun, lastRun, begin, end, or JSON,\n" - "or increasing the tolerance (in dataset.py) from %s.") % (self.__name, firstrunB, lastrunB, tolerance) - except TypeError: - try: - if "unknown" in firstrunB: - return firstrunB - else: - return lastrunB - except TypeError: - return lastrunB - - @cache - def __getFileInfoList( self, dasLimit, parent = False ): - if self.__predefined: - if parent: - extendstring = "secFiles.extend" - else: - extendstring = "readFiles.extend" - with open(self.__fileName) as f: - files = [] - copy = False - for line in f.readlines(): - if "]" in line: - copy = False - if copy: - files.append({name: line.translate(None, "', " + '"')}) - if extendstring in line and "[" in line and "]" not in line: - copy = True - return files - - if parent: - searchdataset = self.parentDataset() - else: - searchdataset = self.__name - dasQuery_files = ( 'file dataset=%s instance=%s detail=true | grep file.name, file.nevents, ' - 'file.creation_time, ' - 'file.modification_time'%( searchdataset, self.__dasinstance ) ) - print("Requesting file information for '%s' from DAS..."%( searchdataset ), end=' ') - sys.stdout.flush() - data = self.__getData( dasQuery_files, dasLimit ) - print("Done.") - data = [ self.__findInJson(entry,"file") for entry in data ] - if len( data ) == 0: - msg = ("No files are available for the dataset '%s'. This can be " - "due to a typo or due to a DAS problem. Please check the " - "spelling of the dataset and/or retry to run " - "'validateAlignments.py'."%( self.name() )) - raise AllInOneError( msg ) - fileInformationList = [] - for file in data: - fileName = 'unknown' - try: - fileName = self.__findInJson(file, "name") - fileCreationTime = self.__findInJson(file, "creation_time") - fileNEvents = self.__findInJson(file, "nevents") - except KeyError: - print(("DAS query gives bad output for file '%s'. Skipping it.\n" - "It may work if you try again later.") % fileName) - fileNEvents = 0 - # select only non-empty files - if fileNEvents == 0: - continue - fileDict = { "name": fileName, - "creation_time": fileCreationTime, - "nevents": fileNEvents - } - fileInformationList.append( fileDict ) - fileInformationList.sort( key=lambda info: self.__findInJson(info,"name") ) - return fileInformationList - - @cache - def __getRunList( self ): - dasQuery_runs = ( 'run dataset=%s instance=%s | grep run.run_number,' - 'run.creation_time'%( self.__name, self.__dasinstance ) ) - print("Requesting run information for '%s' from DAS..."%( self.__name ), end=' ') - sys.stdout.flush() - data = self.__getData( dasQuery_runs ) - print("Done.") - data = [ self.__findInJson(entry,"run") for entry in data ] - data.sort( key = lambda run: self.__findInJson(run, "run_number") ) - return data - - def __datetime(self, stringForDas): - if len(stringForDas) != 8: - raise AllInOneError(stringForDas + " is not a valid date string.\n" - + "DAS accepts dates in the form 'yyyymmdd'") - year = stringForDas[:4] - month = stringForDas[4:6] - day = stringForDas[6:8] - return datetime.date(int(year), int(month), int(day)) - - def __dateString(self, date): - return str(date.year) + str(date.month).zfill(2) + str(date.day).zfill(2) - - def convertTimeToRun( self, begin = None, end = None, - firstRun = None, lastRun = None, - shortTuple = True ): - if ( begin and firstRun ) or ( end and lastRun ): - msg = ( "The Usage of " - + "'begin' & 'firstRun' " * int( bool( begin and - firstRun ) ) - + "and " * int( bool( ( begin and firstRun ) and - ( end and lastRun ) ) ) - + "'end' & 'lastRun' " * int( bool( end and lastRun ) ) - + "is ambigous." ) - raise AllInOneError( msg ) - - if begin or end: - runList = [ self.__findInJson(run, "run_number") for run in self.__getRunList() ] - - if begin: - lastdate = begin - for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months after begin - firstdate = lastdate - lastdate = self.__dateString(self.__datetime(firstdate) + datetime.timedelta(delta)) - dasQuery_begin = "run date between[%s,%s] instance=%s" % (firstdate, lastdate, self.__dasinstance) - begindata = self.__getData(dasQuery_begin) - if len(begindata) > 0: - begindata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"])) - try: - runIndex = self.__find_ge( runList, self.__findInJson(begindata[0], ["run", "run_number"])) - except ValueError: - msg = ( "Your 'begin' is after the creation time of the last " - "run in the dataset\n'%s'"%( self.__name ) ) - raise AllInOneError( msg ) - firstRun = runList[runIndex] - begin = None - break - - if begin: - raise AllInOneError("No runs within a reasonable time interval after your 'begin'." - "Try using a 'begin' that has runs soon after it (within 2 months at most)") - - if end: - firstdate = end - for delta in [ 1, 5, 10, 20, 30 ]: #try searching for about 2 months before end - lastdate = firstdate - firstdate = self.__dateString(self.__datetime(lastdate) - datetime.timedelta(delta)) - dasQuery_end = "run date between[%s,%s] instance=%s" % (firstdate, lastdate, self.__dasinstance) - enddata = self.__getData(dasQuery_end) - if len(enddata) > 0: - enddata.sort(key = lambda run: self.__findInJson(run, ["run", "run_number"])) - try: - runIndex = self.__find_lt( runList, self.__findInJson(enddata[-1], ["run", "run_number"])) - except ValueError: - msg = ( "Your 'end' is before the creation time of the first " - "run in the dataset\n'%s'"%( self.__name ) ) - raise AllInOneError( msg ) - lastRun = runList[runIndex] - end = None - break - - if end: - raise AllInOneError("No runs within a reasonable time interval before your 'end'." - "Try using an 'end' that has runs soon before it (within 2 months at most)") - - if shortTuple: - return firstRun, lastRun - else: - return begin, end, firstRun, lastRun - - def dataType( self ): - if not self.__dataType: - self.__dataType = self.__getDataType() - return self.__dataType - - def magneticField( self ): - if not self.__magneticField: - self.__magneticField = self.__getMagneticField() - return self.__magneticField - - def magneticFieldForRun( self, run = -1 ): - return self.__getMagneticFieldForRun(run) - - def parentDataset( self ): - if not self.__parentDataset: - self.__parentDataset = self.__getParentDataset() - return self.__parentDataset - - def datasetSnippet( self, jsonPath = None, begin = None, end = None, - firstRun = None, lastRun = None, crab = False, parent = False ): - if not firstRun: firstRun = None - if not lastRun: lastRun = None - if not begin: begin = None - if not end: end = None - if self.__predefined and (jsonPath or begin or end or firstRun or lastRun): - msg = ( "The parameters 'JSON', 'begin', 'end', 'firstRun', and 'lastRun' " - "only work for official datasets, not predefined _cff.py files" ) - raise AllInOneError( msg ) - if self.__predefined and parent: - with open(self.__filename) as f: - if "secFiles.extend" not in f.read(): - msg = ("The predefined dataset '%s' does not contain secondary files, " - "which your validation requires!") % self.__name - if self.__official: - self.__name = self.__origName - self.__predefined = False - print(msg) - print ("Retreiving the files from DAS. You will be asked if you want " - "to overwrite the old dataset.\n" - "It will still be compatible with validations that don't need secondary files.") - else: - raise AllInOneError(msg) - - if self.__predefined: - snippet = ("process.load(\"Alignment.OfflineValidation.%s_cff\")\n" - "process.maxEvents = cms.untracked.PSet(\n" - " input = cms.untracked.int32(int(.oO[nEvents]Oo. / .oO[parallelJobs]Oo.))\n" - ")\n" - "process.source.skipEvents=cms.untracked.uint32(int(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.))" - %(self.__name)) - if not parent: - with open(self.__filename) as f: - if "secFiles.extend" in f.read(): - snippet += "\nprocess.source.secondaryFileNames = cms.untracked.vstring()" - return snippet - theMap = { "process": "process.", - "tab": " " * len( "process." ), - "nEvents": ".oO[nEvents]Oo. / .oO[parallelJobs]Oo.", - "skipEventsString": "process.source.skipEvents=cms.untracked.uint32(int(.oO[nIndex]Oo.*.oO[nEvents]Oo./.oO[parallelJobs]Oo.))\n", - "importCms": "", - "header": "" - } - datasetSnippet = self.__createSnippet( jsonPath = jsonPath, - begin = begin, - end = end, - firstRun = firstRun, - lastRun = lastRun, - repMap = theMap, - crab = crab, - parent = parent ) - if jsonPath == "" and begin == "" and end == "" and firstRun == "" and lastRun == "": - try: - self.dump_cff(parent = parent) - except AllInOneError as e: - print("Can't store the dataset as a cff:") - print(e) - print("This may be inconvenient in the future, but will not cause a problem for this validation.") - return datasetSnippet - - @cache - def dump_cff( self, outName = None, jsonPath = None, begin = None, - end = None, firstRun = None, lastRun = None, parent = False ): - if outName == None: - outName = "Dataset" + self.__name.replace("/", "_") - packageName = os.path.join( "Alignment", "OfflineValidation" ) - if not os.path.exists( os.path.join( - self.__cmssw, "src", packageName ) ): - msg = ("You try to store the predefined dataset'%s'.\n" - "For that you need to check out the package '%s' to your " - "private relase area in\n"%( outName, packageName ) - + self.__cmssw ) - raise AllInOneError( msg ) - theMap = { "process": "", - "tab": "", - "nEvents": str( -1 ), - "skipEventsString": "", - "importCms": "import FWCore.ParameterSet.Config as cms\n", - "header": "#Do not delete or (unless you know what you're doing) change these comments\n" - "#%(name)s\n" - "#data type: %(dataType)s\n" - "#magnetic field: .oO[magneticField]Oo.\n" #put in magnetic field later - %{"name": self.__name, #need to create the snippet before getting the magnetic field - "dataType": self.__dataType} #so that we know the first and last runs - } - dataset_cff = self.__createSnippet( jsonPath = jsonPath, - begin = begin, - end = end, - firstRun = firstRun, - lastRun = lastRun, - repMap = theMap, - parent = parent) - magneticField = self.__magneticField - if magneticField == "MagneticField": - magneticField = "%s, %s #%s" % (magneticField, - str(self.__getMagneticFieldForRun()).replace("\n"," ").split("#")[0].strip(), - "Use MagneticField_cff.py; the number is for determining which track selection to use." - ) - dataset_cff = dataset_cff.replace(".oO[magneticField]Oo.",magneticField) - filePath = os.path.join( self.__cmssw, "src", packageName, - "python", outName + "_cff.py" ) - if os.path.exists( filePath ): - existMsg = "The predefined dataset '%s' already exists.\n"%( outName ) - askString = "Do you want to overwrite it? [y/n]\n" - inputQuery = existMsg + askString - while True: - userInput = raw_input( inputQuery ).lower() - if userInput == "y": - break - elif userInput == "n": - return - else: - inputQuery = askString - print ( "The predefined dataset '%s' will be stored in the file\n" - %( outName ) - + filePath + - "\nFor future use you have to do 'scram b'." ) - print() - theFile = open( filePath, "w" ) - theFile.write( dataset_cff ) - theFile.close() - return - - def createdatasetfile_hippy(self, filename, filesperjob, firstrun, lastrun): - with open(filename, "w") as f: - for job in self.__chunks(self.fileList(firstRun=firstrun, lastRun=lastrun, forcerunselection=True), filesperjob): - f.write(",".join("'{}'".format(file) for file in job)+"\n") - - @staticmethod - def getrunnumberfromfilename(filename): - parts = filename.split("/") - result = error = None - if parts[0] != "" or parts[1] != "store": - error = "does not start with /store" - elif parts[2] in ["mc", "relval"]: - result = 1 - elif not parts[-1].endswith(".root"): - error = "does not end with something.root" - elif len(parts) != 12: - error = "should be exactly 11 slashes counting the first one" - else: - runnumberparts = parts[-5:-2] - if not all(len(part)==3 for part in runnumberparts): - error = "the 3 directories {} do not have length 3 each".format("/".join(runnumberparts)) - try: - result = int("".join(runnumberparts)) - except ValueError: - error = "the 3 directories {} do not form an integer".format("/".join(runnumberparts)) - - if error: - error = "could not figure out which run number this file is from:\n{}\n{}".format(filename, error) - raise AllInOneError(error) - - return result - - @cache - def fileList(self, parent=False, firstRun=None, lastRun=None, forcerunselection=False): - fileList = [ self.__findInJson(fileInfo,"name") - for fileInfo in self.fileInfoList(parent) ] - - if firstRun or lastRun: - if not firstRun: firstRun = -1 - if not lastRun: lastRun = float('infinity') - unknownfilenames, reasons = [], set() - for filename in fileList[:]: - try: - if not firstRun <= self.getrunnumberfromfilename(filename) <= lastRun: - fileList.remove(filename) - except AllInOneError as e: - if forcerunselection: raise - unknownfilenames.append(e.message.split("\n")[1]) - reasons .add (e.message.split("\n")[2]) - if reasons: - if len(unknownfilenames) == len(fileList): - print("Could not figure out the run numbers of any of the filenames for the following reason(s):") - else: - print("Could not figure out the run numbers of the following filenames:") - for filename in unknownfilenames: - print(" "+filename) - print("for the following reason(s):") - for reason in reasons: - print(" "+reason) - print("Using the files anyway. The runs will be filtered at the CMSSW level.") - return fileList - - def fileInfoList( self, parent = False ): - return self.__getFileInfoList( self.__dasLimit, parent ) - - def name( self ): - return self.__name - - def predefined( self ): - return self.__predefined - - @cache - def runList( self ): - return self.__getRunList() - - -if __name__ == '__main__': - print("Start testing...") - datasetName = '/MinimumBias/Run2012D-TkAlMinBias-v1/ALCARECO' - jsonFile = ( '/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/' - 'Collisions12/8TeV/Prompt/' - 'Cert_190456-207898_8TeV_PromptReco_Collisions12_JSON.txt' ) - dataset = Dataset( datasetName ) - print(dataset.datasetSnippet( jsonPath = jsonFile, - firstRun = "207800", - end = "20121128")) - dataset.dump_cff( outName = "Dataset_Test_TkAlMinBias_Run2012D", - jsonPath = jsonFile, - firstRun = "207800", - end = "20121128" ) diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/genericValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/genericValidation.py deleted file mode 100644 index 1049eb68d0270..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/genericValidation.py +++ /dev/null @@ -1,794 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import -from builtins import range -from abc import ABCMeta, abstractmethod, abstractproperty -import os -import re -import json -from . import globalDictionaries -from . import configTemplates -from .dataset import Dataset -from .helperFunctions import replaceByMap, addIndex, getCommandOutput2, boolfromstring, pythonboolstring -from .TkAlExceptions import AllInOneError - -class ValidationMetaClass(ABCMeta): - sets = ["mandatories", "optionals", "needpackages"] - dicts = ["defaults"] - def __new__(cls, clsname, bases, dct): - for setname in cls.sets: - if setname not in dct: dct[setname] = set() - dct[setname] = set.union(dct[setname], *(getattr(base, setname) for base in bases if hasattr(base, setname))) - - for dictname in cls.dicts: - if dictname not in dct: dct[dictname] = {} - for base in bases: - if not hasattr(base, dictname): continue - newdict = getattr(base, dictname) - for key in set(newdict) & set(dct[dictname]): - if newdict[key] != dct[dictname][key]: - raise ValueError("Inconsistent values of defaults[{}]: {}, {}".format(key, newdict[key], dct[dictname][key])) - dct[dictname].update(newdict) - - for setname in cls.sets: #e.g. removemandatories, used in preexistingvalidation - #use with caution - if "remove"+setname not in dct: dct["remove"+setname] = set() - dct["remove"+setname] = set.union(dct["remove"+setname], *(getattr(base, "remove"+setname) for base in bases if hasattr(base, "remove"+setname))) - - dct[setname] -= dct["remove"+setname] - - return super(ValidationMetaClass, cls).__new__(cls, clsname, bases, dct) - -class GenericValidation(object, metaclass=ValidationMetaClass): - defaultReferenceName = "DEFAULT" - mandatories = set() - defaults = { - "cmssw": os.environ['CMSSW_BASE'], - "parallelJobs": "1", - "jobid": "", - "needsproxy": "false", - } - needpackages = {"Alignment/OfflineValidation"} - optionals = {"jobmode"} - - def __init__(self, valName, alignment, config): - import random - self.name = valName - self.alignmentToValidate = alignment - self.general = config.getGeneral() - self.randomWorkdirPart = "%0i"%random.randint(1,10e9) - self.configFiles = [] - self.config = config - self.jobid = "" - - theUpdate = config.getResultingSection(self.valType+":"+self.name, - defaultDict = self.defaults, - demandPars = self.mandatories) - self.general.update(theUpdate) - self.jobmode = self.general["jobmode"] - self.NJobs = int(self.general["parallelJobs"]) - self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy") - - # limit maximum number of parallel jobs to 40 - # (each output file is approximately 20MB) - maximumNumberJobs = 40 - if self.NJobs > maximumNumberJobs: - msg = ("Maximum allowed number of parallel jobs " - +str(maximumNumberJobs)+" exceeded!!!") - raise AllInOneError(msg) - if self.NJobs > 1 and not isinstance(self, ParallelValidation): - raise AllInOneError("Parallel jobs not implemented for {}!\n" - "Please set parallelJobs = 1.".format(type(self).__name__)) - - self.jobid = self.general["jobid"] - if self.jobid: - try: #make sure it's actually a valid jobid - output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general) - if "is not found" in output: raise RuntimeError - except RuntimeError: - raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid) - - self.cmssw = self.general["cmssw"] - badcharacters = r"\'" - for character in badcharacters: - if character in self.cmssw: - raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n" - "path name. If you really have it in such a ridiculously named location,\n" - "try making a symbolic link somewhere with a decent name.") - try: - os.listdir(self.cmssw) - except OSError: - raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist') - - if self.cmssw == os.environ["CMSSW_BASE"]: - self.scramarch = os.environ["SCRAM_ARCH"] - self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"] - else: - command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`" - ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"') - commandoutput = getCommandOutput2(command).split('\n') - self.cmssw = commandoutput[0] - self.scramarch = commandoutput[1] - self.cmsswreleasebase = commandoutput[2] - - self.packages = {} - for package in self.needpackages: - for placetolook in self.cmssw, self.cmsswreleasebase: - pkgpath = os.path.join(placetolook, "src", package) - if os.path.exists(pkgpath): - self.packages[package] = pkgpath - break - else: - raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase)) - - self.AutoAlternates = True - if config.has_option("alternateTemplates","AutoAlternates"): - try: - self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower()) - except ValueError: - raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates")) - - knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals - ignoreOpts = [] - config.checkInput(self.valType+":"+self.name, - knownSimpleOptions = knownOpts, - ignoreOptions = ignoreOpts) - - def getRepMap(self, alignment = None): - from .plottingOptions import PlottingOptions - if alignment == None: - alignment = self.alignmentToValidate - try: - result = PlottingOptions(self.config, self.valType) - except KeyError: - result = {} - result.update(alignment.getRepMap()) - result.update(self.general) - result.update({ - "workdir": os.path.join(self.general["workdir"], - self.randomWorkdirPart), - "datadir": self.general["datadir"], - "logdir": self.general["logdir"], - "CommandLineTemplate": ("#run configfile and post-proccess it\n" - "cmsRun %(cfgFile)s\n" - "%(postProcess)s "), - "CMSSW_BASE": self.cmssw, - "SCRAM_ARCH": self.scramarch, - "CMSSW_RELEASE_BASE": self.cmsswreleasebase, - "alignmentName": alignment.name, - "condLoad": alignment.getConditions(), - "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate, - }) - result.update(self.packages) - return result - - @abstractproperty - def filesToCompare(self): - pass - - def getCompareStrings( self, requestId = None, plain = False ): - result = {} - repMap = self.getRepMap().copy() - for validationId in self.filesToCompare: - repMap["file"] = self.filesToCompare[ validationId ] - if repMap["file"].startswith( "/castor/" ): - repMap["file"] = "rfio:%(file)s"%repMap - elif repMap["file"].startswith( "/store/" ): - repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap - if plain: - result[validationId]=repMap["file"] - else: - result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap - if requestId == None: - return result - else: - if not "." in requestId: - requestId += ".%s"%self.defaultReferenceName - if not requestId.split(".")[-1] in result: - msg = ("could not find %s in reference Objects!" - %requestId.split(".")[-1]) - raise AllInOneError(msg) - return result[ requestId.split(".")[-1] ] - - def createFiles(self, fileContents, path, repMap = None, repMaps = None): - """repMap: single map for all files - repMaps: a dict, with the filenames as the keys""" - if repMap is not None and repMaps is not None: - raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both") - result = [] - for fileName in fileContents: - filePath = os.path.join(path, fileName) - result.append(filePath) - - for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)): - theFile = open( filePathi, "w" ) - fileContentsi = fileContents[ fileName ] - if repMaps is not None: - repMap = repMaps[fileName] - if repMap is not None: - repMap.update({"nIndex": str(i)}) - fileContentsi = replaceByMap(fileContentsi, repMap) - theFile.write( fileContentsi ) - theFile.close() - - return result - - def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None): - self.configFiles = self.createFiles(fileContents, - path, repMap = repMap, repMaps = repMaps) - if not schedule == None: - schedule = [os.path.join( path, cfgName) for cfgName in schedule] - for cfgName in schedule: - if not cfgName in self.configFiles: - msg = ("scheduled %s missing in generated configfiles: %s" - %(cfgName, self.configFiles)) - raise AllInOneError(msg) - for cfgName in self.configFiles: - if not cfgName in schedule: - msg = ("generated configuration %s not scheduled: %s" - %(cfgName, schedule)) - raise AllInOneError(msg) - self.configFiles = schedule - return self.configFiles - - def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None): - self.scriptFiles = self.createFiles(fileContents, - path, repMap = repMap, repMaps = repMaps) - for script in self.scriptFiles: - for scriptwithindex in addIndex(script, self.NJobs): - os.chmod(scriptwithindex,0o755) - return self.scriptFiles - - def createCrabCfg(self, fileContents, path ): - if self.NJobs > 1: - msg = ("jobmode 'crab' not supported for parallel validation." - " Please set parallelJobs = 1.") - raise AllInOneError(msg) - self.crabConfigFiles = self.createFiles(fileContents, path) - return self.crabConfigFiles - - -class GenericValidationData(GenericValidation): - """ - Subclass of `GenericValidation` which is the base for validations using - datasets. - """ - needParentFiles = False - mandatories = {"dataset", "maxevents"} - defaults = { - "runRange": "", - "firstRun": "", - "lastRun": "", - "begin": "", - "end": "", - "JSON": "", - "dasinstance": "prod/global", - "ttrhbuilder":"WithAngleAndTemplate", - "usepixelqualityflag": "True", - } - optionals = {"magneticfield"} - - def __init__(self, valName, alignment, config): - """ - This method adds additional items to the `self.general` dictionary - which are only needed for validations using datasets. - - Arguments: - - `valName`: String which identifies individual validation instances - - `alignment`: `Alignment` instance to validate - - `config`: `BetterConfigParser` instance which includes the - configuration of the validations - """ - - super(GenericValidationData, self).__init__(valName, alignment, config) - - # if maxevents is not specified, cannot calculate number of events for - # each parallel job, and therefore running only a single job - if int( self.general["maxevents"] ) < 0 and self.NJobs > 1: - msg = ("Maximum number of events (maxevents) not specified: " - "cannot use parallel jobs.") - raise AllInOneError(msg) - if int( self.general["maxevents"] ) / self.NJobs != float( self.general["maxevents"] ) / self.NJobs: - msg = ("maxevents has to be divisible by parallelJobs") - raise AllInOneError(msg) - - tryPredefinedFirst = (not self.jobmode.split( ',' )[0] == "crab" and self.general["JSON"] == "" - and self.general["firstRun"] == "" and self.general["lastRun"] == "" - and self.general["begin"] == "" and self.general["end"] == "") - - if self.general["dataset"] not in globalDictionaries.usedDatasets: - globalDictionaries.usedDatasets[self.general["dataset"]] = {} - - if self.cmssw not in globalDictionaries.usedDatasets[self.general["dataset"]]: - if globalDictionaries.usedDatasets[self.general["dataset"]] != {}: - print(("Warning: you use the same dataset '%s' in multiple cmssw releases.\n" - "This is allowed, but make sure it's not a mistake") % self.general["dataset"]) - globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw] = {False: None, True: None} - - Bfield = self.general.get("magneticfield", None) - if globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] is None: - dataset = Dataset( - self.general["dataset"], tryPredefinedFirst = tryPredefinedFirst, - cmssw = self.cmssw, cmsswrelease = self.cmsswreleasebase, magneticfield = Bfield, - dasinstance = self.general["dasinstance"]) - globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] = dataset - if tryPredefinedFirst and not dataset.predefined(): #No point finding the data twice in that case - globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][False] = dataset - - self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] - self.general["magneticField"] = self.dataset.magneticField() - self.general["defaultMagneticField"] = "MagneticField" - if self.general["magneticField"] == "unknown": - print("Could not get the magnetic field for this dataset.") - print("Using the default: ", self.general["defaultMagneticField"]) - self.general["magneticField"] = '.oO[defaultMagneticField]Oo.' - - if not self.jobmode.split( ',' )[0] == "crab": - try: - self.general["datasetDefinition"] = self.dataset.datasetSnippet( - jsonPath = self.general["JSON"], - firstRun = self.general["firstRun"], - lastRun = self.general["lastRun"], - begin = self.general["begin"], - end = self.general["end"], - parent = self.needParentFiles ) - except AllInOneError as e: - msg = "In section [%s:%s]: "%(self.valType, self.name) - msg += str(e) - raise AllInOneError(msg) - else: - if self.dataset.predefined(): - msg = ("For jobmode 'crab' you cannot use predefined datasets " - "(in your case: '%s')."%( self.dataset.name() )) - raise AllInOneError( msg ) - try: - theUpdate = config.getResultingSection(self.valType+":"+self.name, - demandPars = ["parallelJobs"]) - except AllInOneError as e: - msg = str(e)[:-1]+" when using 'jobmode: crab'." - raise AllInOneError(msg) - self.general.update(theUpdate) - if self.general["begin"] or self.general["end"]: - ( self.general["begin"], - self.general["end"], - self.general["firstRun"], - self.general["lastRun"] ) = self.dataset.convertTimeToRun( - firstRun = self.general["firstRun"], - lastRun = self.general["lastRun"], - begin = self.general["begin"], - end = self.general["end"], - shortTuple = False) - if self.general["begin"] == None: - self.general["begin"] = "" - if self.general["end"] == None: - self.general["end"] = "" - self.general["firstRun"] = str( self.general["firstRun"] ) - self.general["lastRun"] = str( self.general["lastRun"] ) - if ( not self.general["firstRun"] ) and \ - ( self.general["end"] or self.general["lastRun"] ): - self.general["firstRun"] = str( - self.dataset.runList()[0]["run_number"]) - if ( not self.general["lastRun"] ) and \ - ( self.general["begin"] or self.general["firstRun"] ): - self.general["lastRun"] = str( - self.dataset.runList()[-1]["run_number"]) - if self.general["firstRun"] and self.general["lastRun"]: - if int(self.general["firstRun"]) > int(self.general["lastRun"]): - msg = ( "The lower time/runrange limit ('begin'/'firstRun') " - "chosen is greater than the upper time/runrange limit " - "('end'/'lastRun').") - raise AllInOneError( msg ) - self.general["runRange"] = (self.general["firstRun"] - + '-' + self.general["lastRun"]) - try: - self.general["datasetDefinition"] = self.dataset.datasetSnippet( - jsonPath = self.general["JSON"], - firstRun = self.general["firstRun"], - lastRun = self.general["lastRun"], - begin = self.general["begin"], - end = self.general["end"], - crab = True ) - except AllInOneError as e: - msg = "In section [%s:%s]: "%(self.valType, self.name) - msg += str( e ) - raise AllInOneError( msg ) - - self.general["usepixelqualityflag"] = pythonboolstring(self.general["usepixelqualityflag"], "usepixelqualityflag") - - def getRepMap(self, alignment = None): - result = super(GenericValidationData, self).getRepMap(alignment) - outputfile = os.path.expandvars(replaceByMap( - "%s_%s_.oO[name]Oo..root" % (self.outputBaseName, self.name) - , result)) - resultfile = os.path.expandvars(replaceByMap(("/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./" + - "%s_%s_.oO[name]Oo..root" % (self.resultBaseName, self.name)) - , result)) - result.update({ - "resultFile": ".oO[resultFiles[.oO[nIndex]Oo.]]Oo.", - "resultFiles": addIndex(resultfile, self.NJobs), - "finalResultFile": resultfile, - "outputFile": ".oO[outputFiles[.oO[nIndex]Oo.]]Oo.", - "outputFiles": addIndex(outputfile, self.NJobs), - "finalOutputFile": outputfile, - "ProcessName": self.ProcessName, - "Bookkeeping": self.Bookkeeping, - "LoadBasicModules": self.LoadBasicModules, - "TrackSelectionRefitting": self.TrackSelectionRefitting, - "ValidationConfig": self.ValidationTemplate, - "FileOutputTemplate": self.FileOutputTemplate, - "DefinePath": self.DefinePath, - }) - return result - - @property - def cfgName(self): - return "%s.%s.%s_cfg.py"%( self.configBaseName, self.name, - self.alignmentToValidate.name ) - @abstractproperty - def ProcessName(self): - pass - - @property - def cfgTemplate(self): - return configTemplates.cfgTemplate - - @abstractproperty - def ValidationTemplate(self): - pass - - @property - def filesToCompare(self): - return {self.defaultReferenceName: self.getRepMap()["finalResultFile"]} - - def createConfiguration(self, path ): - repMap = self.getRepMap() - cfgs = {self.cfgName: self.cfgTemplate} - super(GenericValidationData, self).createConfiguration(cfgs, path, repMap=repMap) - - def createScript(self, path, template = configTemplates.scriptTemplate, downloadFiles=[], repMap = None, repMaps = None): - scriptName = "%s.%s.%s.sh"%(self.scriptBaseName, self.name, - self.alignmentToValidate.name ) - if repMap is None and repMaps is None: - repMap = self.getRepMap() - repMap["CommandLine"]="" - for cfg in self.configFiles: - repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":addIndex(cfg, self.NJobs, ".oO[nIndex]Oo."), - "postProcess":"" - } - scripts = {scriptName: template} - return super(GenericValidationData, self).createScript(scripts, path, downloadFiles = downloadFiles, - repMap = repMap, repMaps = repMaps) - - def createCrabCfg(self, path, crabCfgBaseName): - """ - Method which creates a `crab.cfg` for a validation on datasets. - - Arguments: - - `path`: Path at which the file will be stored. - - `crabCfgBaseName`: String which depends on the actual type of - validation calling this method. - """ - crabCfgName = "crab.%s.%s.%s.cfg"%( crabCfgBaseName, self.name, - self.alignmentToValidate.name ) - repMap = self.getRepMap() - repMap["script"] = "dummy_script.sh" - # repMap["crabOutputDir"] = os.path.basename( path ) - repMap["crabWorkingDir"] = crabCfgName.split( '.cfg' )[0] - self.crabWorkingDir = repMap["crabWorkingDir"] - repMap["numberOfJobs"] = self.general["parallelJobs"] - repMap["cfgFile"] = self.configFiles[0] - repMap["queue"] = self.jobmode.split( ',' )[1].split( '-q' )[1] - if self.dataset.dataType() == "mc": - repMap["McOrData"] = "events = .oO[nEvents]Oo." - elif self.dataset.dataType() == "data": - repMap["McOrData"] = "lumis = -1" - if self.jobmode.split( ',' )[0] == "crab": - print ("For jobmode 'crab' the parameter 'maxevents' will be " - "ignored and all events will be processed.") - else: - raise AllInOneError("Unknown data type! Can't run in crab mode") - crabCfg = {crabCfgName: replaceByMap( configTemplates.crabCfgTemplate, - repMap ) } - return super(GenericValidationData, self).createCrabCfg( crabCfg, path ) - - @property - def Bookkeeping(self): - return configTemplates.Bookkeeping - @property - def LoadBasicModules(self): - return configTemplates.LoadBasicModules - @abstractproperty - def TrackSelectionRefitting(self): - pass - @property - def FileOutputTemplate(self): - return configTemplates.FileOutputTemplate - @abstractproperty - def DefinePath(self): - pass - -class GenericValidationData_CTSR(GenericValidationData): - #common track selection and refitting - defaults = { - "momentumconstraint": "None", - "openmasswindow": "False", - "cosmicsdecomode": "True", - "removetrackhitfiltercommands": "", - "appendtrackhitfiltercommands": "", - } - def getRepMap(self, alignment=None): - result = super(GenericValidationData_CTSR, self).getRepMap(alignment) - - from .trackSplittingValidation import TrackSplittingValidation - result.update({ - "ValidationSequence": self.ValidationSequence, - "istracksplitting": str(isinstance(self, TrackSplittingValidation)), - "cosmics0T": str(self.cosmics0T), - "use_d0cut": str(self.use_d0cut), - "ispvvalidation": str(self.isPVValidation) - }) - - commands = [] - for removeorappend in "remove", "append": - optionname = removeorappend + "trackhitfiltercommands" - if result[optionname]: - for command in result[optionname].split(","): - command = command.strip() - commands.append('process.TrackerTrackHitFilter.commands.{}("{}")'.format(removeorappend, command)) - result["trackhitfiltercommands"] = "\n".join(commands) - - return result - @property - def use_d0cut(self): - return "Cosmics" not in self.general["trackcollection"] #use it for collisions only - @property - def isPVValidation(self): - return False # only for PV Validation sequence - @property - def TrackSelectionRefitting(self): - return configTemplates.CommonTrackSelectionRefitting - @property - def DefinePath(self): - return configTemplates.DefinePath_CommonSelectionRefitting - @abstractproperty - def ValidationSequence(self): - pass - @property - def cosmics0T(self): - if "Cosmics" not in self.general["trackcollection"]: return False - Bfield = self.dataset.magneticFieldForRun() - if Bfield < 0.5: return True - if isinstance(Bfield, str): - if "unknown " in Bfield: - msg = Bfield.replace("unknown ","",1) - elif Bfield == "unknown": - msg = "Can't get the B field for %s." % self.dataset.name() - else: - msg = "B field = {}???".format(Bfield) - raise AllInOneError(msg + "\n" - "To use this dataset, specify magneticfield = [value] in your .ini config file.") - return False - -class ParallelValidation(GenericValidation): - @classmethod - def initMerge(cls): - return "" - @abstractmethod - def appendToMerge(self): - pass - - @classmethod - def doInitMerge(cls): - from .plottingOptions import PlottingOptions - result = cls.initMerge() - result = replaceByMap(result, PlottingOptions(None, cls)) - if result and result[-1] != "\n": result += "\n" - return result - def doMerge(self): - result = self.appendToMerge() - if result[-1] != "\n": result += "\n" - result += ("if [[ tmpMergeRetCode -eq 0 ]]; then\n" - " xrdcp -f .oO[finalOutputFile]Oo. root://eoscms//eos/cms.oO[finalResultFile]Oo.\n" - "fi\n" - "if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then\n" - " mergeRetCode=${tmpMergeRetCode}\n" - "fi\n") - result = replaceByMap(result, self.getRepMap()) - return result - -class ValidationWithPlots(GenericValidation): - @classmethod - def runPlots(cls, validations): - return ("cp .oO[plottingscriptpath]Oo. .\n" - "root -x -b -q .oO[plottingscriptname]Oo.++") - @abstractmethod - def appendToPlots(self): - pass - @abstractmethod - def plottingscriptname(cls): - """override with a classmethod""" - @abstractmethod - def plottingscripttemplate(cls): - """override with a classmethod""" - @abstractmethod - def plotsdirname(cls): - """override with a classmethod""" - - @classmethod - def doRunPlots(cls, validations): - from .plottingOptions import PlottingOptions - cls.createPlottingScript(validations) - result = cls.runPlots(validations) - result = replaceByMap(result, PlottingOptions(None, cls)) - if result and result[-1] != "\n": result += "\n" - return result - @classmethod - def createPlottingScript(cls, validations): - from .plottingOptions import PlottingOptions - repmap = PlottingOptions(None, cls).copy() - filename = replaceByMap(".oO[plottingscriptpath]Oo.", repmap) - repmap["PlottingInstantiation"] = "\n".join( - replaceByMap(v.appendToPlots(), v.getRepMap()).rstrip("\n") - for v in validations - ) - plottingscript = replaceByMap(cls.plottingscripttemplate(), repmap) - with open(filename, 'w') as f: - f.write(plottingscript) - -class ValidationWithPlotsSummaryBase(ValidationWithPlots): - class SummaryItem(object): - def __init__(self, name, values, format=None, latexname=None, latexformat=None): - """ - name: name of the summary item, goes on top of the column - values: value for each alignment (in order of rows) - format: python format string (default: {:.3g}, meaning up to 3 significant digits) - latexname: name in latex form, e.g. if name=sigma you might want latexname=\sigma (default: name) - latexformat: format for latex (default: format) - """ - if format is None: format = "{:.3g}" - if latexname is None: latexname = name - if latexformat is None: latexformat = format - - self.__name = name - self.__values = values - self.__format = format - self.__latexname = latexname - self.__latexformat = latexformat - - def name(self, latex=False): - if latex: - return self.__latexname - else: - return self.__name - - def format(self, value, latex=False): - if latex: - fmt = self.__latexformat - else: - fmt = self.__format - if re.match(".*[{][^}]*[fg][}].*", fmt): - value = float(value) - return fmt.format(value) - - def values(self, latex=False): - result = [self.format(v, latex=latex) for v in self.__values] - return result - - def value(self, i, latex): - return self.values(latex)[i] - - @abstractmethod - def getsummaryitems(cls, folder): - """override with a classmethod that returns a list of SummaryItems - based on the plots saved in folder""" - - __summaryitems = None - __lastfolder = None - - @classmethod - def summaryitemsstring(cls, folder=None, latex=False, transpose=True): - if folder is None: folder = cls.plotsdirname() - if folder.startswith( "/castor/" ): - folder = "rfio:%(file)s"%repMap - elif folder.startswith( "/store/" ): - folder = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap - - if cls.__summaryitems is None or cls.__lastfolder != folder: - cls.__lastfolder = folder - cls.__summaryitems = cls.getsummaryitems(folder) - - summaryitems = cls.__summaryitems - - if not summaryitems: - raise AllInOneError("No summary items!") - size = {len(_.values(latex)) for _ in summaryitems} - if len(size) != 1: - raise AllInOneError("Some summary items have different numbers of values\n{}".format(size)) - size = size.pop() - - if transpose: - columnwidths = ([max(len(_.name(latex)) for _ in summaryitems)] - + [max(len(_.value(i, latex)) for _ in summaryitems) for i in range(size)]) - else: - columnwidths = [max(len(entry) for entry in [_.name(latex)] + _.values(latex)) for _ in summaryitems] - - if latex: - join = " & " - else: - join = " " - row = join.join("{{:{}}}".format(width) for width in columnwidths) - - if transpose: - rows = [row.format(*[_.name(latex)]+_.values(latex)) for _ in summaryitems] - else: - rows = [] - rows.append(row.format(*(_.name for _ in summaryitems))) - for i in range(size): - rows.append(row.format(*(_.value(i, latex) for _ in summaryitems))) - - if latex: - join = " \\\\\n" - else: - join = "\n" - result = join.join(rows) - if latex: - result = (r"\begin{{tabular}}{{{}}}".format("|" + "|".join("c"*(len(columnwidths))) + "|") + "\n" - + result + "\n" - + r"\end{tabular}") - return result - - @classmethod - def printsummaryitems(cls, *args, **kwargs): - print(cls.summaryitemsstring(*args, **kwargs)) - @classmethod - def writesummaryitems(cls, filename, *args, **kwargs): - with open(filename, "w") as f: - f.write(cls.summaryitemsstring(*args, **kwargs)+"\n") - -class ValidationWithPlotsSummary(ValidationWithPlotsSummaryBase): - @classmethod - def getsummaryitems(cls, folder): - result = [] - with open(os.path.join(folder, "{}Summary.txt".format(cls.__name__))) as f: - for line in f: - split = line.rstrip("\n").split("\t") - kwargs = {} - for thing in split[:]: - if thing.startswith("format="): - kwargs["format"] = thing.replace("format=", "", 1) - split.remove(thing) - if thing.startswith("latexname="): - kwargs["latexname"] = thing.replace("latexname=", "", 1) - split.remove(thing) - if thing.startswith("latexformat="): - kwargs["latexformat"] = thing.replace("latexformat=", "", 1) - split.remove(thing) - - name = split[0] - values = split[1:] - result.append(cls.SummaryItem(name, values, **kwargs)) - return result - -class ValidationWithComparison(GenericValidation): - @classmethod - def doComparison(cls, validations): - from .plottingOptions import PlottingOptions - repmap = PlottingOptions(None, cls).copy() - repmap["compareStrings"] = " , ".join(v.getCompareStrings("OfflineValidation") for v in validations) - repmap["compareStringsPlain"] = " , ".join(v.getCompareStrings("OfflineValidation", True) for v in validations) - comparison = replaceByMap(cls.comparisontemplate(), repmap) - return comparison - - @classmethod - def comparisontemplate(cls): - return configTemplates.compareAlignmentsExecution - @classmethod - def comparealignmentspath(cls): - return ".oO[Alignment/OfflineValidation]Oo./scripts/.oO[compareAlignmentsName]Oo." - @abstractmethod - def comparealignmentsname(cls): - """classmethod""" - -class ValidationForPresentation(ValidationWithPlots): - @abstractmethod - def presentationsubsections(cls): - """classmethod""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparison.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparison.py deleted file mode 100644 index c77a462a827d3..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparison.py +++ /dev/null @@ -1,340 +0,0 @@ -from __future__ import absolute_import -import os -import configparser as ConfigParser # needed for exceptions in this module -from . import configTemplates -from .genericValidation import GenericValidation -from .helperFunctions import replaceByMap, getCommandOutput2, cppboolstring, pythonboolstring, clean_name -from .TkAlExceptions import AllInOneError - - -class GeometryComparison(GenericValidation): - """ - Object representing a geometry comparison job. - """ - defaults = { - "3DSubdetector1":"1", - "3DSubdetector2":"2", - "3DTranslationalScaleFactor":"50", - "modulesToPlot":"all", - "moduleList": "./CREATE_NEW/emptyModuleList.txt", - "useDefaultRange":"false", - "plotOnlyGlobal":"true", - "plotPng":"true", - "makeProfilePlots":"true", - "dx_min":"-99999", - "dx_max":"-99999", - "dy_min":"-99999", - "dy_max":"-99999", - "dz_min":"-99999", - "dz_max":"-99999", - "dr_min":"-99999", - "dr_max":"-99999", - "rdphi_min":"-99999", - "rdphi_max":"-99999", - "dalpha_min":"-99999", - "dalpha_max":"-99999", - "dbeta_min":"-99999", - "dbeta_max":"-99999", - "dgamma_min":"-99999", - "dgamma_max":"-99999", - "multiIOV":"False", - } - mandatories = {"levels", "dbOutput"} - valType = "compare" - def __init__( self, valName, alignment, referenceAlignment, - config, copyImages = True): - """ - Constructor of the GeometryComparison class. - - Arguments: - - `valName`: String which identifies individual validation instances - - `alignment`: `Alignment` instance to validate - - `referenceAlignment`: `Alignment` instance which is compared - with `alignment` - - `config`: `BetterConfigParser` instance which includes the - configuration of the validations - - `copyImages`: Boolean which indicates whether png- and pdf-files - should be copied back from the batch farm - """ - super(GeometryComparison, self).__init__(valName, alignment, config) - self.referenceAlignment = referenceAlignment - referenceName = "IDEAL" - if not self.referenceAlignment == "IDEAL": - referenceName = self.referenceAlignment.name - - allCompares = config.getCompares() - self.__compares = {} - self.__filesToCompare = {} - if valName in allCompares: - self.__compares[valName] = allCompares[valName] - else: - msg = ("Could not find compare section '%s' in '%s'" - %(valName, allCompares)) - raise AllInOneError(msg) - self.copyImages = copyImages - - for name in "useDefaultRange", "plotOnlyGlobal", "plotPng": - self.general[name] = cppboolstring(self.general[name], name) - - - def getRepMap(self, alignment = None): - if alignment == None: - alignment = self.alignmentToValidate - repMap = super(GeometryComparison, self).getRepMap( alignment ) - referenceName = "IDEAL" - referenceTitle = "IDEAL" - if not self.referenceAlignment == "IDEAL": - referenceName = self.referenceAlignment.name - referenceTitle = self.referenceAlignment.title - - assert len(self.__compares) == 1 #? not sure how it can be anything else, but just in case - common = list(self.__compares.keys())[0] - - repMap.update({ - "common": clean_name(common), - "comparedGeometry": (".oO[alignmentName]Oo." - "ROOTGeometry.root"), - "referenceGeometry": "IDEAL", # will be replaced later - # if not compared to IDEAL - "reference": clean_name(referenceName), - "referenceTitle": referenceTitle, - "alignmentTitle": self.alignmentToValidate.title, - "moduleListBase": os.path.basename(repMap["moduleList"]), - }) - if not referenceName == "IDEAL": - repMap["referenceGeometry"] = (".oO[reference]Oo." - "ROOTGeometry.root") - repMap["name"] += "_vs_.oO[reference]Oo." - return repMap - - @property - def filesToCompare(self): - return self.__filesToCompare - - def createConfiguration(self, path ): - # self.__compares - repMap = self.getRepMap() - cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%( - self.alignmentToValidate.name) - cfgs = {cfgFileName: configTemplates.intoNTuplesTemplate} - repMaps = {cfgFileName: repMap} - if not self.referenceAlignment == "IDEAL": - referenceRepMap = self.getRepMap( self.referenceAlignment ) - cfgFileName = "TkAlCompareToNTuple.%s_cfg.py"%( - self.referenceAlignment.name ) - cfgs[cfgFileName] = configTemplates.intoNTuplesTemplate - repMaps[cfgFileName] = referenceRepMap - - cfgSchedule = list(cfgs.keys()) - for common in self.__compares: - repMap.update({ - "levels": self.__compares[common][0], - "dbOutput": pythonboolstring(self.__compares[common][1], "dbOutput") - }) - if self.__compares[common][1].split()[0] == "true": - repMap["dbOutputService"] = configTemplates.dbOutputTemplate - else: - repMap["dbOutputService"] = "" - cfgName = replaceByMap(("TkAlCompareCommon.oO[common]Oo.." - ".oO[name]Oo._cfg.py"),repMap) - cfgs[cfgName] = configTemplates.compareTemplate - repMaps[cfgName] = repMap - - cfgSchedule.append( cfgName ) - super(GeometryComparison, self).createConfiguration(cfgs, path, cfgSchedule, repMaps = repMaps) - - def createScript(self, path): - repMap = self.getRepMap() - repMap["runComparisonScripts"] = "" - scriptName = replaceByMap(("TkAlGeomCompare.%s..oO[name]Oo..sh" - %self.name), repMap) - - y_ranges = "" - plottedDifferences = ["dx","dy","dz","dr","rdphi","dalpha","dbeta","dgamma"] - for diff in plottedDifferences: - y_ranges += ","+repMap["%s_min"%diff] - y_ranges += ","+repMap["%s_max"%diff] - - for name in self.__compares: - if '"DetUnit"' in self.__compares[name][0].split(","): - repMap["outputFile"] = (".oO[name]Oo..Comparison_common"+name+".root") - repMap["nIndex"] = ("") - repMap["runComparisonScripts"] += \ - ("cp .oO[Alignment/OfflineValidation]Oo." - "/scripts/comparisonScript.C .\n" - "cp .oO[Alignment/OfflineValidation]Oo." - "/scripts/GeometryComparisonPlotter.h .\n" - "cp .oO[Alignment/OfflineValidation]Oo." - "/scripts/GeometryComparisonPlotter.cc .\n" - "root -b -q 'comparisonScript.C+(\"" - ".oO[name]Oo..Comparison_common"+name+".root\",\"" - "./\",\".oO[modulesToPlot]Oo.\",\".oO[alignmentName]Oo.\",\".oO[reference]Oo.\",.oO[useDefaultRange]Oo.,.oO[plotOnlyGlobal]Oo.,.oO[plotPng]Oo.,.oO[makeProfilePlots]Oo."+y_ranges+")'\n" - "cp "+path+"/TkAl3DVisualization_.oO[common]Oo._.oO[name]Oo..C .\n" - "root -l -b -q TkAl3DVisualization_.oO[common]Oo._.oO[name]Oo..C+\n") - if self.copyImages: - repMap["runComparisonScripts"] += \ - ("mkdir -p .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/Translations\n") - repMap["runComparisonScripts"] += \ - ("mkdir -p .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/Rotations\n") - - - ### At the moment translations are images with suffix _1 and _2, rotations _3 and _4 - ### The numeration depends on the order of the MakePlots(x, y) commands in comparisonScript.C - ### If comparisonScript.C is changed, check if the following lines need to be changed as well - - if repMap["plotPng"] == "true": - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_1*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Translations/\" \n") - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_2*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Translations/\" \n") - - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_3*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Rotations/\" \n") - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_4*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Rotations/\" \n") - - else: - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_1*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Translations/\" \n") - - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name \"*_2*\" " - "-print | xargs -I {} bash -c \"cp {} .oO[datadir]Oo." - "/.oO[name]Oo..Comparison_common"+name+"_Images/Rotations/\" \n") - - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name " - "\"*.tex\" -print | xargs -I {} bash -c" - " \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/\" \n") - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name " - "\"TkMap_SurfDeform*.pdf\" -print | xargs -I {} bash -c" - " \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/\" \n") - repMap["runComparisonScripts"] += \ - ("find . -maxdepth 1 -name " - "\"TkMap_SurfDeform*.png\" -print | xargs -I {} bash -c" - " \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/\" \n") - repMap["runComparisonScripts"] += \ - ("cp .oO[Alignment/OfflineValidation]Oo." - "/macros/makeArrowPlots.C " - ".\n" - "root -b -q 'makeArrowPlots.C(\"" - ".oO[name]Oo..Comparison_common"+name - +".root\",\".oO[name]Oo.." - +name+"_ArrowPlots\")'\n") - repMap["runComparisonScripts"] += \ - ("mkdir -p .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/ArrowPlots\n") - repMap["runComparisonScripts"] += \ - ("find .oO[name]Oo.."+name+"_ArrowPlots " - "-maxdepth 1 -name \"*.png\" -print | xargs -I {} bash " - "-c \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/ArrowPlots\"\n") - repMap["runComparisonScripts"] += \ - ("find .oO[name]Oo.."+name+"_ArrowPlots " - "-maxdepth 1 -name \"*.pdf\" -print | xargs -I {} bash " - "-c \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/ArrowPlots\"\n") - repMap["runComparisonScripts"] += \ - ("find . " - "-maxdepth 1 -name \".oO[common]Oo._.oO[name]Oo..Visualization_rotated.gif\" -print | xargs -I {} bash " - "-c \"cp {} .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/.oO[common]Oo._.oO[name]Oo..Visualization.gif\"\n") - - # TkAlMap inFile=tree.root compAl=UL2018 refAl=StartGeom savePNG=True TkVersion=phase1 outDir=./test_plots/tanh colPal=2 - range_str = '' - plottedDifferences = ["dx","dy","dz","dr","rdphi","dalpha","dbeta","dgamma"] - for diff in plottedDifferences: - range_str += diff+'_range=['+str(repMap[diff+'_min'])+','+str(repMap[diff+'_max'])+'];' - repMap["runComparisonScripts"] += \ - ("mkdir -p .oO[datadir]Oo./.oO[name]Oo." - ".Comparison_common"+name+"_Images/TkAlMapPlots\n") - repMap["runComparisonScripts"] += \ - ("python .oO[Alignment/OfflineValidation]Oo./python/runGCPTkAlMap.py -b " - "inFile=.oO[name]Oo..Comparison_common"+name+".root " - "refAl=\".oO[reference]Oo.\" " - "compAl=\".oO[alignmentName]Oo.\" " - "savePNG=True " - "TkVersion=\"phase0\" " - "colPal=2 " - "defRanges=\""+range_str+"\" " - "outDir=.oO[datadir]Oo./.oO[name]Oo..Comparison_common"+name+"_Images/TkAlMapPlots\n") - #"outDir=.oO[name]Oo.."+name+"_TkMapPlots " - #"useDefaultRanges=.oO[useDefaultRange]Oo. "+range_str+"\n") - - # Copy root file for check - repMap["runComparisonScripts"] += \ - ("cp .oO[name]Oo..Comparison_common"+name+".root " - ".oO[datadir]Oo./.oO[name]Oo..Comparison_common"+name+"_Images/TkAlMapPlots/GCP.root\n") - #repMap["runComparisonScripts"] += \ - # ("cp .oO[alignmentName]Oo.ROOTGeometry.root " - # ".oO[datadir]Oo./.oO[name]Oo..Comparison_common"+name+"_Images/TkAlMapPlots/comparedGeometry.root\n") - - resultingFile = replaceByMap(("/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./compared%s_" - ".oO[name]Oo..root"%name), repMap) - resultingFile = os.path.expandvars( resultingFile ) - resultingFile = os.path.abspath( resultingFile ) - resultingFile = "root://eoscms//eos/cms" + resultingFile #needs to be AFTER abspath so that it doesn't eat the // - self.__filesToCompare[ name ] = resultingFile - - else: - raise AllInOneError("Need to have DetUnit in levels!") - - repMap["CommandLine"]="" - repMap["CommandLine"]+= \ - "# copy module list required for comparison script \n" - if repMap["moduleList"].startswith("/store"): - repMap["CommandLine"]+= \ - "xrdcp root://eoscms//eos/cms.oO[moduleList]Oo. .\n" - elif repMap["moduleList"].startswith("root://"): - repMap["CommandLine"]+= \ - "xrdcp .oO[moduleList]Oo. .\n" - elif repMap["moduleList"].startswith("./CREATE_NEW/"): - repMap["CommandLine"]+= \ - "touch .oO[moduleListBase]Oo.\n" - else: - repMap["CommandLine"]+= \ - "cp .oO[moduleList]Oo. .\n" - - try: - getCommandOutput2(replaceByMap("cd $(mktemp -d)\n.oO[CommandLine]Oo.\ncat .oO[moduleListBase]Oo.", repMap)) - except RuntimeError: - raise AllInOneError(replaceByMap(".oO[moduleList]Oo. does not exist!", repMap)) - - for cfg in self.configFiles: - # FIXME: produce this line only for enabled dbOutput - # postProcess = "cp .oO[workdir]Oo./*.db .oO[datadir]Oo.\n" - # postProcess = "cp *.db .oO[datadir]Oo.\n" - postProcess = "" - repMap["CommandLine"]+= \ - repMap["CommandLineTemplate"]%{"cfgFile":cfg, - "postProcess":postProcess} - repMap["CommandLine"]+= ("# overall postprocessing\n" - ".oO[runComparisonScripts]Oo.\n" - ) - - #~ print configTemplates.scriptTemplate - scripts = {scriptName: replaceByMap( configTemplates.scriptTemplate, repMap )} - files = {replaceByMap("TkAl3DVisualization_.oO[common]Oo._.oO[name]Oo..C", repMap ): replaceByMap(configTemplates.visualizationTrackerTemplate, repMap )} - self.createFiles(files, path) - return super(GeometryComparison, self).createScript(scripts, path) - - def createCrabCfg(self, path): - msg = ("Parallelization not supported for geometry comparison. Please " - "choose another 'jobmode'.") - raise AllInOneError(msg) diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparisonTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparisonTemplates.py deleted file mode 100644 index c14ea1878118f..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/geometryComparisonTemplates.py +++ /dev/null @@ -1,175 +0,0 @@ -###################################################################### -###################################################################### -intoNTuplesTemplate=""" -import FWCore.ParameterSet.Config as cms - -process = cms.Process("ValidationIntoNTuples") - -.oO[LoadGlobalTagTemplate]Oo. - -process.load("Configuration.Geometry.GeometryRecoDB_cff") - -process.load("CondCore.CondDB.CondDB_cfi") - -process.MessageLogger = cms.Service("MessageLogger", - destinations = cms.untracked.vstring('detailedInfo', - 'cout') -) - -.oO[condLoad]Oo. - -process.source = cms.Source("EmptySource", - firstRun=cms.untracked.uint32(.oO[runGeomComp]Oo.) - ) - -process.maxEvents = cms.untracked.PSet( - input = cms.untracked.int32(1) -) -process.dump = cms.EDAnalyzer("TrackerGeometryIntoNtuples", - outputFile = cms.untracked.string('.oO[alignmentName]Oo.ROOTGeometry.root'), - outputTreename = cms.untracked.string('alignTree') -) - -process.p = cms.Path(process.dump) -""" - - -###################################################################### -###################################################################### -compareTemplate=""" -import FWCore.ParameterSet.Config as cms - -process = cms.Process("validation") - -.oO[LoadGlobalTagTemplate]Oo. - -process.load("Configuration.Geometry.GeometryRecoDB_cff") - -process.load("CondCore.CondDB.CondDB_cfi") - -process.MessageLogger = cms.Service("MessageLogger", - destinations = cms.untracked.vstring('detailedInfo', - 'cout') -) - -process.source = cms.Source("EmptySource", - firstRun=cms.untracked.uint32(.oO[runGeomComp]Oo.) - ) - -process.maxEvents = cms.untracked.PSet( - input = cms.untracked.int32(1) -) - -process.siStripQualityESProducer.ListOfRecordToMerge=cms.VPSet( - cms.PSet(record = cms.string('SiStripDetCablingRcd'), - tag = cms.string('')), - cms.PSet(record = cms.string('RunInfoRcd'), - tag = cms.string('')), - cms.PSet(record = cms.string('SiStripBadChannelRcd'), - tag = cms.string('')), - cms.PSet(record = cms.string('SiStripBadFiberRcd'), - tag = cms.string('')), - cms.PSet(record = cms.string('SiStripBadModuleRcd'), - tag = cms.string('')), - cms.PSet(record = cms.string('SiStripBadStripRcd'), - tag = cms.string('')) -) - -process.load("DQM.SiStripCommon.TkHistoMap_cff") - - # configuration of the Tracker Geometry Comparison Tool - # Tracker Geometry Comparison -process.load("Alignment.OfflineValidation.TrackerGeometryCompare_cfi") - # the input "IDEAL" is special indicating to use the ideal geometry of the release - -process.TrackerGeometryCompare.inputROOTFile1 = '.oO[referenceGeometry]Oo.' -process.TrackerGeometryCompare.inputROOTFile2 = '.oO[comparedGeometry]Oo.' -process.TrackerGeometryCompare.moduleList = '.oO[moduleListBase]Oo.' -process.TrackerGeometryCompare.outputFile = ".oO[name]Oo..Comparison_common.oO[common]Oo..root" - -process.load("CommonTools.UtilAlgos.TFileService_cfi") -process.TFileService.fileName = cms.string("TkSurfDeform_.oO[name]Oo..Comparison_common.oO[common]Oo..root") - -process.TrackerGeometryCompare.levels = [ .oO[levels]Oo. ] - - ##FIXME!!!!!!!!! - ##replace TrackerGeometryCompare.writeToDB = .oO[dbOutput]Oo. - ##removed: dbOutputService - -process.p = cms.Path(process.TrackerGeometryCompare) -""" - - -###################################################################### -###################################################################### -dbOutputTemplate= """ -//_________________________ db Output ____________________________ - # setup for writing out to DB - include "CondCore/DBCommon/CondDBSetup.cfi" -# include "CondCore/DBCommon/data/CondDBCommon.cfi" - - service = PoolDBOutputService { - using CondDBSetup - VPSet toPut = { - { string record = "TrackerAlignmentRcd" string tag = ".oO[tag]Oo." }, - { string record = "TrackerAlignmentErrorExtendedRcd" string tag = ".oO[errortag]Oo." } - } - # string connect = "sqlite_file:.oO[workdir]Oo./.oO[name]Oo.Common.oO[common]Oo..db" - string connect = "sqlite_file:.oO[name]Oo.Common.oO[common]Oo..db" - # untracked string catalog = "file:alignments.xml" - untracked string timetype = "runnumber" - } -""" - -###################################################################### -###################################################################### -visualizationTrackerTemplate= """ -#include "Alignment/OfflineValidation/scripts/visualizationTracker.C" -void TkAl3DVisualization_.oO[common]Oo._.oO[name]Oo.(){ - //------------------------------ONLY NEEDED INPUTS-------------------------------// -//------Tree Read In-------- - TString inputFileName = ".oO[outputFile]Oo."; - //output file name - string outputFileName = ".oO[common]Oo._.oO[name]Oo..Visualization"; - //title - string line1 = ".oO[alignmentTitle]Oo."; - string line2 = "vs. .oO[referenceTitle]Oo."; - //set subdetectors to see - int subdetector1 = .oO[3DSubdetector1]Oo.; - int subdetector2 = .oO[3DSubdetector2]Oo.; - //translation scale factor - int sclftr = .oO[3DTranslationalScaleFactor]Oo.; - //rotation scale factor - int sclfrt = 1; - //module size scale factor - float sclfmodulesizex = 1; - float sclfmodulesizey = 1; - float sclfmodulesizez = 1; - //beam pipe radius - float piperadius = 2.25; - //beam pipe xy coordinates - float pipexcoord = 0; - float pipeycoord = 0; - //beam line xy coordinates - float linexcoord = 0; - float lineycoord = 0; -//------------------------------End of ONLY NEEDED INPUTS-------------------------------// - cout << "running visualizer" << endl; - runVisualizer(inputFileName, - outputFileName, - line1, - line2, - subdetector1, - subdetector2, - sclftr, - sclfrt, - sclfmodulesizex, - sclfmodulesizey, - sclfmodulesizez, - piperadius, - pipexcoord, - pipeycoord, - linexcoord, - lineycoord ); -} -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/globalDictionaries.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/globalDictionaries.py deleted file mode 100644 index e89b8be6dc262..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/globalDictionaries.py +++ /dev/null @@ -1,7 +0,0 @@ -# Store used datasets, to avoid making the same DAS query multiple times -usedDatasets = {} - -# Needed for more than one geometry comparison for one alignment -alignRandDict = {} - -plottingOptions = {} diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/helperFunctions.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/helperFunctions.py deleted file mode 100644 index 459894b721337..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/helperFunctions.py +++ /dev/null @@ -1,225 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import -from builtins import range -import os -import re -import ROOT -import sys -from .TkAlExceptions import AllInOneError -import CondCore.Utilities.conddblib as conddblib - -####################--- Helpers ---############################ -def replaceByMap(target, the_map): - """This function replaces `.oO[key]Oo.` by `the_map[key]` in target. - - Arguments: - - `target`: String which contains symbolic tags of the form `.oO[key]Oo.` - - `the_map`: Dictionary which has to contain the `key`s in `target` as keys - """ - - result = target - for key in the_map: - lifeSaver = 10e3 - iteration = 0 - while ".oO[" in result and "]Oo." in result: - for key in the_map: - try: - result = result.replace(".oO["+key+"]Oo.",the_map[key]) - except TypeError: #try a dict - try: - for keykey, value in the_map[key].items(): - result = result.replace(".oO[" + key + "['" + keykey + "']]Oo.", value) - result = result.replace(".oO[" + key + '["' + keykey + '"]]Oo.', value) - except AttributeError: #try a list - try: - for index, value in enumerate(the_map[key]): - result = result.replace(".oO[" + key + "[" + str(index) + "]]Oo.", value) - except TypeError: - raise TypeError("Something is wrong in replaceByMap! Need a string, dict, or list, but the_map(%s)=%s!"%(repr(key), repr(the_map[key]))) - iteration += 1 - if iteration > lifeSaver: - problematicLines = "" - for line in result.splitlines(): - if ".oO[" in result and "]Oo." in line: - problematicLines += "%s\n"%line - msg = ("Oh Dear, there seems to be an endless loop in " - "replaceByMap!!\n%s\n%s"%(problematicLines, the_map)) - raise AllInOneError(msg) - return result - - -def getCommandOutput2(command): - """This function executes `command` and returns it output. - - Arguments: - - `command`: Shell command to be invoked by this function. - """ - - child = os.popen(command) - data = child.read() - err = child.close() - if err: - raise RuntimeError('%s failed w/ exit code %d' % (command, err)) - return data - - -def castorDirExists(path): - """This function checks if the directory given by `path` exists. - - Arguments: - - `path`: Path to castor directory - """ - - if path[-1] == "/": - path = path[:-1] - containingPath = os.path.join( *path.split("/")[:-1] ) - dirInQuestion = path.split("/")[-1] - try: - rawLines = getCommandOutput2("rfdir /"+containingPath).splitlines() - except RuntimeError: - return False - for line in rawLines: - if line.split()[0][0] == "d": - if line.split()[8] == dirInQuestion: - return True - return False - -def replacelast(string, old, new, count = 1): - """Replace the last occurances of a string""" - return new.join(string.rsplit(old,count)) - -fileExtensions = ["_cfg.py", ".sh", ".root"] - -def addIndex(filename, njobs, index = None): - if index is None: - return [addIndex(filename, njobs, i) for i in range(njobs)] - if njobs == 1: - return filename - - fileExtension = None - for extension in fileExtensions: - if filename.endswith(extension): - fileExtension = extension - if fileExtension is None: - raise AllInOneError(fileName + " does not end with any of the extensions " - + str(fileExtensions)) - return replacelast(filename, fileExtension, "_" + str(index) + fileExtension) - -def parsecolor(color): - try: #simplest case: it's an int - return int(color) - except ValueError: - pass - - try: #kRed, kBlue, ... - color = str(getattr(ROOT, color)) - return int(color) - except (AttributeError, ValueError): - pass - - if color.count("+") + color.count("-") == 1: #kRed+5, kGreen-2 - if "+" in color: #don't want to deal with nonassociativity of - - split = color.split("+") - color1 = parsecolor(split[0]) - color2 = parsecolor(split[1]) - return color1 + color2 - - if "-" in color: - split = color.split("-") - color1 = parsecolor(split[0]) - color2 = parsecolor(split[1]) - return color1 - color2 - - raise AllInOneError("color has to be an integer, a ROOT constant (kRed, kBlue, ...), or a two-term sum or difference (kGreen-5)!") - -def parsestyle(style): - try: #simplest case: it's an int - return int(style) - except ValueError: - pass - - try: #kStar, kDot, ... - style = str(getattr(ROOT,style)) - return int(style) - except (AttributeError, ValueError): - pass - - raise AllInOneError("style has to be an integer or a ROOT constant (kDashed, kStar, ...)!") - -def recursivesubclasses(cls): - result = [cls] - for subcls in cls.__subclasses__(): - result += recursivesubclasses(subcls) - return result - -def cache(function): - cache = {} - def newfunction(*args, **kwargs): - try: - return cache[args, tuple(sorted(kwargs.items()))] - except TypeError: - print(args, tuple(sorted(kwargs.items()))) - raise - except KeyError: - cache[args, tuple(sorted(kwargs.items()))] = function(*args, **kwargs) - return newfunction(*args, **kwargs) - newfunction.__name__ = function.__name__ - return newfunction - -def boolfromstring(string, name): - """ - Takes a string from the configuration file - and makes it into a bool - """ - #try as a string, not case sensitive - if string.lower() == "true": return True - if string.lower() == "false": return False - #try as a number - try: - return str(bool(int(string))) - except ValueError: - pass - #out of options - raise ValueError("{} has to be true or false!".format(name)) - - -def pythonboolstring(string, name): - """ - Takes a string from the configuration file - and makes it into a bool string for a python template - """ - return str(boolfromstring(string, name)) - -def cppboolstring(string, name): - """ - Takes a string from the configuration file - and makes it into a bool string for a C++ template - """ - return pythonboolstring(string, name).lower() - -def getTagsMap(db): - con = conddblib.connect(url = conddblib.make_url(db)) - session = con.session() - TAG = session.get_dbtype(conddblib.Tag) - dictionary = {} - for i in range(0,len(session.query(TAG.object_type).order_by(TAG.name).all())): - q1 = session.query(TAG.object_type).order_by(TAG.name).all()[i][0] - q2 = session.query(TAG.name).order_by(TAG.name).all()[i][0] - dictionary[q1]=q2 - - return dictionary - -def clean_name(s): - """Transforms a string into a valid variable or method name. - - Arguments: - - `s`: input string - """ - - # Remove invalid characters - s = re.sub(r"[^0-9a-zA-Z_]", "", s) - - # Remove leading characters until we find a letter or underscore - s = re.sub(r"^[^a-zA-Z_]+", "", s) - - return s diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidation.py deleted file mode 100644 index d4a3cc80bb533..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidation.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import absolute_import -import os -from . import configTemplates -from . import globalDictionaries -from .dataset import Dataset -from .genericValidation import GenericValidationData -from .helperFunctions import replaceByMap -from .TkAlExceptions import AllInOneError - - -class MonteCarloValidation(GenericValidationData): - configBaseName = "TkAlMcValidate" - scriptBaseName = "TkAlMcValidate" - crabCfgBaseName = "TkAlMcValidate" - resultBaseName = "McValidation" - outputBaseName = "McValidation" - needParentFiles = True - valType = "mcValidate" - def __init__(self, valName, alignment, config): - super(MonteCarloValidation, self).__init__(valName, alignment, config) - if self.NJobs > 1: - raise AllInOneError("Parallel jobs not implemented for the MC validation!\n" - "Please set parallelJobs = 1.") - - @property - def cfgTemplate(self): - return configTemplates.mcValidateTemplate - - def createScript(self, path): - return super(MonteCarloValidation, self).createScript(path) - - def createCrabCfg(self, path): - return super(MonteCarloValidation, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap( self, alignment = None ): - repMap = super(MonteCarloValidation, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"] - }) - repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] ) - repMap["resultFile"] = os.path.expandvars( repMap["resultFile"] ) - return repMap - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidationTemplates.py deleted file mode 100644 index 1f3b7831189d5..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/monteCarloValidationTemplates.py +++ /dev/null @@ -1,66 +0,0 @@ -###################################################################### -###################################################################### -mcValidateTemplate=""" -import FWCore.ParameterSet.Config as cms - -process = cms.Process("TkVal") -process.load("FWCore.MessageService.MessageLogger_cfi") -process.MessageLogger = cms.Service("MessageLogger", - destinations = cms.untracked.vstring('LOGFILE_McValidate_.oO[name]Oo.', - 'cout') -) - -### standard includes -process.load('Configuration.Geometry.GeometryPilot2_cff') -process.load("Configuration.StandardSequences.RawToDigi_cff") -process.load("Configuration.EventContent.EventContent_cff") -process.load("Configuration.StandardSequences.Reconstruction_cff") -process.load("Configuration.StandardSequences..oO[magneticField]Oo._cff") -process.load("SimGeneral.MixingModule.mixNoPU_cfi") - -.oO[LoadGlobalTagTemplate]Oo. - -.oO[condLoad]Oo. - - -### validation-specific includes -process.load("SimTracker.TrackAssociatorProducers.trackAssociatorByHits_cfi") -process.load("Validation.RecoTrack.cuts_cff") -process.load("Validation.RecoTrack.MultiTrackValidator_cff") -process.load("SimGeneral.TrackingAnalysis.trackingParticles_cfi") - -### configuration MultiTrackValidator ### -process.multiTrackValidator.outputFile = '.oO[outputFile]Oo.' - -process.multiTrackValidator.associators = ['trackAssociatorByHits'] -process.multiTrackValidator.UseAssociators = cms.bool(True) -process.multiTrackValidator.label = ['generalTracks'] - -.oO[datasetDefinition]Oo. -process.source.inputCommands = cms.untracked.vstring('keep *', 'drop *_MEtoEDMConverter_*_*') # hack to get rid of the memory consumption problem in 2_2_X and beond - -process.options = cms.untracked.PSet( - wantSummary = cms.untracked.bool(False), - Rethrow = cms.untracked.vstring("ProductNotFound"), # make this exception fatal - fileMode = cms.untracked.string('NOMERGE') # no ordering needed, but calls endRun/beginRun etc. at file boundaries -) - -process.re_tracking_and_TP = cms.Sequence(process.mix*process.trackingParticles* - process.siPixelRecHits*process.siStripMatchedRecHits* - process.ckftracks* - process.cutsRecoTracks* - process.trackAssociatorByHits* - process.multiTrackValidator - ) - -process.re_tracking = cms.Sequence(process.siPixelRecHits*process.siStripMatchedRecHits* - process.ckftracks* - process.cutsRecoTracks* - process.trackAssociatorByHits* - process.multiTrackValidator - ) - -### final path and endPath -process.p = cms.Path(process.re_tracking) -""" - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidation.py deleted file mode 100644 index ed677f8a0f5d0..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidation.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import absolute_import -import os -from . import configTemplates -from . import globalDictionaries -from .genericValidation import GenericValidationData_CTSR, ParallelValidation, ValidationWithComparison, ValidationForPresentation, ValidationWithPlots, ValidationWithPlotsSummary -from .helperFunctions import replaceByMap, addIndex, pythonboolstring -from .presentation import SubsectionFromList, SubsectionOnePage -from .TkAlExceptions import AllInOneError - -class OfflineValidation(GenericValidationData_CTSR, ParallelValidation, ValidationWithComparison, ValidationWithPlotsSummary, ValidationForPresentation): - configBaseName = "TkAlOfflineValidation" - scriptBaseName = "TkAlOfflineValidation" - crabCfgBaseName = "TkAlOfflineValidation" - resultBaseName = "AlignmentValidation" - outputBaseName = "AlignmentValidation" - defaults = { - "offlineModuleLevelHistsTransient": "False", - "offlineModuleLevelProfiles": "True", - "stripYResiduals": "False", - "maxtracks": "0", - "chargeCut": "0", - "multiIOV": "False", - } - deprecateddefaults = { - "DMRMethod":"", - "DMRMinimum":"", - "DMROptions":"", - "OfflineTreeBaseDir":"", - "SurfaceShapes":"", - } - defaults.update(deprecateddefaults) - mandatories = {"trackcollection"} - valType = "offline" - - def __init__(self, valName, alignment, config): - super(OfflineValidation, self).__init__(valName, alignment, config) - - for name in "offlineModuleLevelHistsTransient", "offlineModuleLevelProfiles", "stripYResiduals": - self.general[name] = pythonboolstring(self.general[name], name) - - for option in self.deprecateddefaults: - if self.general[option]: - raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option) - del self.general[option] - - if self.NJobs > 1 and self.general["offlineModuleLevelHistsTransient"] == "True": - msg = ("To be able to merge results when running parallel jobs," - " set offlineModuleLevelHistsTransient to false.") - raise AllInOneError(msg) - - try: - self.NTracks = int(self.general["maxtracks"]) - if self.NTracks < 0: raise ValueError - except ValueError: - raise AllInOneError("maxtracks has to be a positive integer, or 0 for no limit") - - if self.NTracks / self.NJobs != float(self.NTracks) / self.NJobs: - raise AllInOneError("maxtracks has to be divisible by parallelJobs") - - @property - def ProcessName(self): - return "OfflineValidator" - - @property - def ValidationTemplate(self): - return configTemplates.offlineTemplate - - @property - def ValidationSequence(self): - return configTemplates.OfflineValidationSequence - - @property - def FileOutputTemplate(self): - return configTemplates.offlineFileOutputTemplate - - def createScript(self, path): - return super(OfflineValidation, self).createScript(path) - - def createCrabCfg(self, path): - return super(OfflineValidation, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap(self, alignment = None): - repMap = super(OfflineValidation, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"], - "offlineValidationMode": "Standalone", - "TrackCollection": self.general["trackcollection"], - "filetoplot": "root://eoscms//eos/cms.oO[finalResultFile]Oo.", - }) - - return repMap - - def appendToPlots(self): - return ' p.loadFileList(".oO[filetoplot]Oo.", ".oO[title]Oo.", .oO[color]Oo., .oO[style]Oo.);\n' - - @classmethod - def initMerge(cls): - from .plottingOptions import PlottingOptions - outFilePath = replaceByMap(".oO[scriptsdir]Oo./TkAlOfflineJobsMerge.C", PlottingOptions(None, cls.valType)) - print("outFilePath") - print(outFilePath) - with open(outFilePath, "w") as theFile: - theFile.write(replaceByMap(configTemplates.mergeOfflineParJobsTemplate, {})) - result = super(OfflineValidation, cls).initMerge() - result += ("cp .oO[Alignment/OfflineValidation]Oo./scripts/merge_TrackerOfflineValidation.C .\n" - "cp .oO[mergeOfflineParJobsScriptPath]Oo. .\n") - return result - - def appendToMerge(self): - repMap = self.getRepMap() - - parameters = "root://eoscms//eos/cms" + ",root://eoscms//eos/cms".join(repMap["resultFiles"]) - - mergedoutputfile = "root://eoscms//eos/cms%(finalResultFile)s"%repMap - return ('root -x -b -q -l "TkAlOfflineJobsMerge.C(\\\"' - +parameters+'\\\",\\\"'+mergedoutputfile+'\\\")"') - - @classmethod - def plottingscriptname(cls): - return "TkAlExtendedOfflineValidation.C" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.extendedValidationTemplate - - @classmethod - def plotsdirname(cls): - return "ExtendedOfflineValidation_Images" - - @classmethod - def comparealignmentsname(cls): - return "compareAlignments.cc" - - @classmethod - def presentationsubsections(cls): - return [ - SubsectionOnePage('chi2', r'$\chi^2$ plots'), - SubsectionSubdetectors('DmedianY*R_[^_]*.eps$', 'DMR'), - SubsectionSubdetectors('DmedianY*R.*plain.eps$', 'DMR'), - SubsectionSubdetectors('DmedianY*R.*split.eps$','Split DMR'), - SubsectionSubdetectors('DrmsNY*R_[^_]*.eps$', 'DRnR'), - SubsectionSubdetectors('DrmsNY*R.*plain.eps$', 'DRnR'), - SubsectionSubdetectors('SurfaceShape', 'Surface Shape'), - ] - -class SubsectionSubdetectors(SubsectionFromList): - pageidentifiers = ( - ("BPIX", "BPIX"), - ("FPIX", "FPIX"), - ("TIB", "TIB"), - ("TID", "TID"), - ("TOB", "TOB"), - ("TEC", "TEC"), - ) - -class OfflineValidationDQM(OfflineValidation): - configBaseName = "TkAlOfflineValidationDQM" - def __init__(self, valName, alignment, config): - super(OfflineValidationDQM, self).__init__(valName, alignment, config) - if not config.has_section("DQM"): - msg = "You need to have a DQM section in your configfile!" - raise AllInOneError(msg) - - self.__PrimaryDataset = config.get("DQM", "primaryDataset") - self.__firstRun = int(config.get("DQM", "firstRun")) - self.__lastRun = int(config.get("DQM", "lastRun")) - - def getRepMap(self, alignment = None): - repMap = super(OfflineValidationDQM, self).getRepMap(alignment) - repMap.update({ - "workdir": os.path.expandvars(repMap["workdir"]), - "offlineValidationMode": "Dqm", - "workflow": ("/%s/TkAl%s-.oO[alignmentName]Oo._R%09i_R%09i_" - "ValSkim-v1/ALCARECO" - %(self.__PrimaryDataset, - datetime.datetime.now().strftime("%y"), - self.__firstRun, self.__lastRun)), - "firstRunNumber": "%i"% self.__firstRun - }) - if "__" in repMap["workflow"]: - msg = ("the DQM workflow specefication must not contain '__'. " - "it is: %s"%repMap["workflow"]) - raise AllInOneError(msg) - return repMap - - @property - def FileOutputTemplate(self): - return configTemplates.offlineDqmFileOutputTemplate diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidationTemplates.py deleted file mode 100644 index 3dd9a759a571a..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/offlineValidationTemplates.py +++ /dev/null @@ -1,125 +0,0 @@ -###################################################################### -###################################################################### -offlineTemplate = """ -process.oneGoodVertexFilter = cms.EDFilter("VertexSelector", - src = cms.InputTag("offlinePrimaryVertices"), - cut = cms.string("!isFake && ndof > 4 && abs(z) <= 15 && position.Rho <= 2"), # tracksSize() > 3 for the older cut - filter = cms.bool(True), # otherwise it won't filter the events, just produce an empty vertex collection. - ) - - - -process.FilterGoodEvents=cms.Sequence(process.oneGoodVertexFilter) - - -process.noScraping= cms.EDFilter("FilterOutScraping", - src=cms.InputTag(".oO[TrackCollection]Oo."), - applyfilter = cms.untracked.bool(True), - debugOn = cms.untracked.bool(False), ## Or 'True' to get some per-event info - numtrack = cms.untracked.uint32(10), - thresh = cms.untracked.double(0.25) - ) -#################################### - -# Use compressions settings of TFile -# see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings -# settings = 100 * algorithm + level -# level is from 1 (small) to 9 (large compression) -# algo: 1 (ZLIB), 2 (LMZA) -# see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance -compressionSettings = 207 - - ## - ## Load and Configure OfflineValidation and Output File - ## -process.load("Alignment.OfflineValidation.TrackerOfflineValidation_.oO[offlineValidationMode]Oo._cff") -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..compressionSettings = compressionSettings -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..Tracks = 'FinalTrackRefitter' -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..trajectoryInput = 'FinalTrackRefitter' -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..moduleLevelHistsTransient = .oO[offlineModuleLevelHistsTransient]Oo. -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..moduleLevelProfiles = .oO[offlineModuleLevelProfiles]Oo. -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..stripYResiduals = .oO[stripYResiduals]Oo. -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..maxTracks = int(.oO[maxtracks]Oo./.oO[parallelJobs]Oo.) -process.TrackerOfflineValidation.oO[offlineValidationMode]Oo..chargeCut = .oO[chargeCut]Oo. -""" - -OfflineValidationSequence = "process.seqTrackerOfflineValidation.oO[offlineValidationMode]Oo." - - -###################################################################### -###################################################################### -mergeOfflineParJobsTemplate=""" -#include "Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C" - -int TkAlOfflineJobsMerge(TString pars, TString outFile) -{ -// load framework lite just to find the CMSSW libs... -gSystem->Load("libFWCoreFWLite"); -FWLiteEnabler::enable(); - -return hadd(pars, outFile); -} -""" - - -###################################################################### -###################################################################### -offlineFileOutputTemplate = """ -process.TFileService.fileName = '.oO[outputFile]Oo.' -""" - - -###################################################################### -###################################################################### -offlineDqmFileOutputTemplate = """ -process.DqmSaverTkAl.workflow = '.oO[workflow]Oo.' -process.DqmSaverTkAl.dirName = '.oO[workdir]Oo./.' -process.DqmSaverTkAl.forceRunNumber = .oO[firstRunNumber]Oo. -""" - - -###################################################################### -###################################################################### -extendedValidationExecution=""" -#run extended offline validation scripts -echo -e "\n\nRunning extended offline validation" - -cp .oO[extendedValScriptPath]Oo. . -root -x -b -q -l TkAlExtendedOfflineValidation.C - -""" - - -###################################################################### -###################################################################### -extendedValidationTemplate=""" -#include "Alignment/OfflineValidation/macros/PlotAlignmentValidation.C" -#include "FWCore/FWLite/interface/FWLiteEnabler.h" - -void TkAlExtendedOfflineValidation() -{ - TkAlStyle::legendheader = ".oO[legendheader]Oo."; - TkAlStyle::legendoptions = ".oO[legendoptions]Oo."; - TkAlStyle::set(.oO[publicationstatus]Oo., .oO[era]Oo., ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo."); - bool bigtext = .oO[bigtext]Oo.; - gStyle->SetTitleH ( 0.07 ); - gStyle->SetTitleW ( 1.00 ); - gStyle->SetTitleFont ( 132 ); - // load framework lite just to find the CMSSW libs... - gSystem->Load("libFWCoreFWLite"); - FWLiteEnabler::enable(); - - PlotAlignmentValidation p(bigtext); -.oO[PlottingInstantiation]Oo. - p.setOutputDir(".oO[datadir]Oo./.oO[PlotsDirName]Oo."); - p.useFitForDMRplots(.oO[usefit]Oo.); - p.setTreeBaseDir(".oO[OfflineTreeBaseDir]Oo."); - p.plotDMR(".oO[DMRMethod]Oo.",.oO[DMRMinimum]Oo.,".oO[DMROptions]Oo."); - p.plotSurfaceShapes(".oO[SurfaceShapes]Oo."); - p.plotChi2("root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./.oO[validationId]Oo._result.root"); - vector moduleids = {.oO[moduleid]Oo.}; - for (auto moduleid : moduleids) { - p.residual_by_moduleID(moduleid); - } -} -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidation.py deleted file mode 100644 index 08d9508266b6d..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidation.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import absolute_import - -import os - -from . import configTemplates -from .genericValidation import GenericValidationData_CTSR, ParallelValidation, ValidationWithPlots -from .helperFunctions import replaceByMap -from .presentation import SubsectionFromList, SubsectionOnePage -from .TkAlExceptions import AllInOneError - - -class OverlapValidation(GenericValidationData_CTSR, ParallelValidation, ValidationWithPlots): - configBaseName = "TkAlOverlapValidation" - scriptBaseName = "TkAlOverlapValidation" - crabCfgBaseName = "TkAlOverlapValidation" - resultBaseName = "OverlapValidation" - outputBaseName = "OverlapValidation" - mandatories = {"trackcollection"} - valType = "overlap" - - @property - def ValidationTemplate(self): - return configTemplates.overlapTemplate - - @property - def ValidationSequence(self): - return configTemplates.overlapValidationSequence - - @property - def ProcessName(self): - return "overlap" - - def getRepMap( self, alignment = None ): - repMap = super(OverlapValidation, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"], - "TrackCollection": self.general["trackcollection"], - }) - return repMap - - def appendToPlots(self): - """ - if no argument or "" is passed a string with an instantiation is - returned, else the validation is appended to the list - """ - return '("{file}", "{title}", {color}, {style}),'.format(file=self.getCompareStrings(plain=True)["DEFAULT"], **self.getRepMap()) - - def appendToMerge(self): - repMap = self.getRepMap() - - parameters = " ".join(os.path.join("root://eoscms//eos/cms", file.lstrip("/")) for file in repMap["resultFiles"]) - - mergedoutputfile = os.path.join("root://eoscms//eos/cms", repMap["finalResultFile"].lstrip("/")) - return "hadd -f %s %s" % (mergedoutputfile, parameters) - - @classmethod - def plottingscriptname(cls): - return "TkAlOverlapValidation.py" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.overlapPlottingTemplate - - @classmethod - def plotsdirname(cls): - return "OverlapValidationPlots" - - @classmethod - def runPlots(cls, validations): - return ("rfcp .oO[plottingscriptpath]Oo. .\n" - "python .oO[plottingscriptname]Oo.") - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidationTemplates.py deleted file mode 100644 index 85f85d69ee1a1..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/overlapValidationTemplates.py +++ /dev/null @@ -1,62 +0,0 @@ -overlapTemplate = """ -# Use compressions settings of TFile -# see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings -# settings = 100 * algorithm + level -# level is from 1 (small) to 9 (large compression) -# algo: 1 (ZLIB), 2 (LMZA) -# see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance -compressionSettings = 207 -process.analysis = cms.EDAnalyzer("OverlapValidation", - usePXB = cms.bool(True), - usePXF = cms.bool(True), - useTIB = cms.bool(True), - useTOB = cms.bool(True), - useTID = cms.bool(True), - useTEC = cms.bool(True), - compressionSettings = cms.untracked.int32(compressionSettings), - ROUList = cms.vstring('TrackerHitsTIBLowTof', - 'TrackerHitsTIBHighTof', - 'TrackerHitsTOBLowTof', - 'TrackerHitsTOBHighTof'), - trajectories = cms.InputTag("FinalTrackRefitter"), - associatePixel = cms.bool(False), - associateStrip = cms.bool(False), - associateRecoTracks = cms.bool(False), - tracks = cms.InputTag("FinalTrackRefitter"), - barrelOnly = cms.bool(False) -) - -""" - -overlapValidationSequence = "process.analysis" - -overlapPlottingTemplate = """ - -import os -import ROOT -from Alignment.OfflineValidation.TkAlStyle import TkAlStyle - -TkAlStyle.legendheader = ".oO[legendheader]Oo." -TkAlStyle.set(ROOT..oO[publicationstatus]Oo., ROOT..oO[era]Oo., ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo.") - -try: - os.makedirs(".oO[datadir]Oo./.oO[PlotsDirName]Oo./") -except OSError: - pass -try: - os.makedirs(".oO[datadir]Oo./.oO[PlotsDirName]Oo./Profiles") -except OSError: - pass - -from Alignment.OfflineValidation.overlapValidationPlot import plot - -subdet_ids=[True,True,True,True,True,True]#(BPIX,FPIX,TIB,TID,TOB,TEC) -module_directions=[True,True,True]#(z,r,phi) -overlap_directions=[True,True,True]#(z,r,phi) -profile_directions=[True,True,True,True]#(histogtam,z-profiles,r-profiles,phi-profiles) - - -plot(".oO[datadir]Oo./.oO[PlotsDirName]Oo./",subdet_ids,module_directions,overlap_directions,profile_directions,.oO[PlottingInstantiation]Oo.) - - -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/plottingOptions.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/plottingOptions.py deleted file mode 100644 index 258734faf4244..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/plottingOptions.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import absolute_import -from builtins import range -import os -import random - -from . import globalDictionaries -from . import configTemplates - -from .genericValidation import ValidationMetaClass, ValidationWithComparison, ValidationWithPlots -from .helperFunctions import getCommandOutput2, replaceByMap, cppboolstring -from .offlineValidation import OfflineValidation -from .primaryVertexValidation import PrimaryVertexValidation -from .primaryVertexResolution import PrimaryVertexResolution -from .TkAlExceptions import AllInOneError -from .trackSplittingValidation import TrackSplittingValidation -from .zMuMuValidation import ZMuMuValidation -from .overlapValidation import OverlapValidation - -class BasePlottingOptions(object, metaclass=ValidationMetaClass): - defaults = { - "cmssw" : os.environ["CMSSW_BASE"], - "publicationstatus" : "", - "customtitle" : "", - "customrighttitle" : "", - "era" : "NONE", - "legendheader" : "", - "legendoptions":"all", - } - mandatories = set() - needpackages = {"Alignment/OfflineValidation"} - def __init__(self, config, valType): - import random - self.type = valType - self.general = config.getGeneral() - self.randomWorkdirPart = "%0i"%random.randint(1,10e9) - self.config = config - - theUpdate = config.getResultingSection("plots:"+self.type, - defaultDict = self.defaults, - demandPars = self.mandatories) - self.general.update(theUpdate) - - self.cmssw = self.general["cmssw"] - badcharacters = r"\'" - for character in badcharacters: - if character in self.cmssw: - raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n" - "path name. If you really have it in such a ridiculously named location,\n" - "try making a symbolic link somewhere with a decent name.") - try: - os.listdir(self.cmssw) - except OSError: - raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist') - - if self.cmssw == os.environ["CMSSW_BASE"]: - self.scramarch = os.environ["SCRAM_ARCH"] - self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"] - else: - command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`" - ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"') - commandoutput = getCommandOutput2(command).split('\n') - self.cmssw = commandoutput[0] - self.scramarch = commandoutput[1] - self.cmsswreleasebase = commandoutput[2] - - for package in self.needpackages: - for placetolook in self.cmssw, self.cmsswreleasebase: - pkgpath = os.path.join(placetolook, "src", package) - if os.path.exists(pkgpath): - self.general[package] = pkgpath - break - else: - raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase)) - - self.general["publicationstatus"] = self.general["publicationstatus"].upper() - self.general["era"] = self.general["era"].upper() - - if not self.general["publicationstatus"] and not self.general["customtitle"]: - self.general["publicationstatus"] = "INTERNAL" - if self.general["customtitle"] and not self.general["publicationstatus"]: - self.general["publicationstatus"] = "CUSTOM" - - if self.general["publicationstatus"] != "CUSTOM" and self.general["customtitle"]: - raise AllInOneError("If you would like to use a custom title, please leave out the 'publicationstatus' parameter") - if self.general["publicationstatus"] == "CUSTOM" and not self.general["customtitle"]: - raise AllInOneError("If you want to use a custom title, you should provide it using 'customtitle' in the [plots:%s] section" % valType) - - if self.general["era"] != "NONE" and self.general["customrighttitle"]: - raise AllInOneError("If you would like to use a custom right title, please leave out the 'era' parameter") - - publicationstatusenum = ["INTERNAL", "INTERNAL_SIMULATION", "PRELIMINARY", "PUBLIC", "SIMULATION", "UNPUBLISHED", "CUSTOM"] - eraenum = ["NONE", "CRUZET15", "CRAFT15", "COLL0T15"] - if self.general["publicationstatus"] not in publicationstatusenum: - raise AllInOneError("Publication status must be one of " + ", ".join(publicationstatusenum) + "!") - if self.general["era"] not in eraenum: - raise AllInOneError("Era must be one of " + ", ".join(eraenum) + "!") - - knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals - ignoreOpts = [] - config.checkInput("plots:"+self.type, - knownSimpleOptions = knownOpts, - ignoreOptions = ignoreOpts) - - def getRepMap(self): - result = self.general - result.update({ - "workdir": os.path.join(self.general["workdir"], - self.randomWorkdirPart), - "datadir": self.general["datadir"], - "logdir": self.general["logdir"], - "CMSSW_BASE": self.cmssw, - "SCRAM_ARCH": self.scramarch, - "CMSSW_RELEASE_BASE": self.cmsswreleasebase, - "validationId": self.validationclass.__name__, - }) - if issubclass(self.validationclass, ValidationWithPlots): - result["plottingscriptname"] = self.validationclass.plottingscriptname() - result["plottingscriptpath"] = ".oO[scriptsdir]Oo./.oO[plottingscriptname]Oo." - result["PlotsDirName"] = self.validationclass.plotsdirname() - if issubclass(self.validationclass, ValidationWithComparison): - result["compareAlignmentsPath"] = self.validationclass.comparealignmentspath() - result["compareAlignmentsName"] = self.validationclass.comparealignmentsname() - return result - -class PlottingOptionsTrackSplitting(BasePlottingOptions): - defaults = { - "outliercut": "-1.0", - "subdetector": "none", - } - needpackages = {"Alignment/CommonAlignmentProducer"} - validationclass = TrackSplittingValidation - def __init__(self, config): - super(PlottingOptionsTrackSplitting, self).__init__(config, "split") - validsubdets = self.validsubdets() - if self.general["subdetector"] not in validsubdets: - raise AllInOneError("'%s' is not a valid subdetector!\n" % self.general["subdetector"] + "The options are: " + ", ".join(validsubdets)) - - def validsubdets(self): - filename = replaceByMap(".oO[Alignment/CommonAlignmentProducer]Oo./python/AlignmentTrackSelector_cfi.py", self.getRepMap()) - with open(filename) as f: - trackselector = f.read() - - minhitspersubdet = trackselector.split("minHitsPerSubDet")[1].split("(",1)[1] - - parenthesesdepth = 0 - i = 0 - for character in minhitspersubdet: - if character == "(": - parenthesesdepth += 1 - if character == ")": - parenthesesdepth -= 1 - if parenthesesdepth < 0: - break - i += 1 - minhitspersubdet = minhitspersubdet[0:i] - - results = minhitspersubdet.split(",") - empty = [] - for i in range(len(results)): - results[i] = results[i].split("=")[0].strip().replace("in", "", 1) - - results.append("none") - - return [a for a in results if a] - -class PlottingOptionsZMuMu(BasePlottingOptions): - defaults = { - "resonance": "Z", - "switchONfit": "false", - "rebinphi": "4", - "rebinetadiff": "2", - "rebineta": "2", - "rebinpt": "8", - "AutoSetRange": "false", - } - needpackages = {"MuonAnalysis/MomentumScaleCalibration"} - validationclass = ZMuMuValidation - def __init__(self, config): - super(PlottingOptionsZMuMu, self).__init__(config, "zmumu") - self.general["switchONfit"] = cppboolstring(self.general["switchONfit"], "switchONfit") - -class PlottingOptionsOffline(BasePlottingOptions): - defaults = { - "DMRMethod":"median,rmsNorm", - "DMRMinimum":"30", - "DMROptions":"", - "OfflineTreeBaseDir":"TrackHitFilter", - "SurfaceShapes":"coarse", - "bigtext":"false", - "mergeOfflineParJobsScriptPath": ".oO[scriptsdir]Oo./TkAlOfflineJobsMerge.C", - "usefit": "false","moduleid": "" - } - validationclass = OfflineValidation - def __init__(self, config): - super(PlottingOptionsOffline, self).__init__(config, "offline") - for name in "usefit", "bigtext": - self.general[name] = cppboolstring(self.general[name], name) - - -class PlottingOptionsPrimaryVertex(BasePlottingOptions): - defaults = { - "autoLimits":"false", - "doMaps":"false", - "stdResiduals":"true", - "m_dxyPhiMax":"40", - "m_dzPhiMax":"40", - "m_dxyEtaMax":"40", - "m_dzEtaMax":"40", - "m_dxyPhiNormMax":"0.5", - "m_dzPhiNormMax":"0.5", - "m_dxyEtaNormMax":"0.5", - "m_dzEtaNormMax":"0.5", - "w_dxyPhiMax":"150", - "w_dzPhiMax":"150", - "w_dxyEtaMax":"150", - "w_dzEtaMax":"1000", - "w_dxyPhiNormMax":"1.8", - "w_dzPhiNormMax":"1.8", - "w_dxyEtaNormMax":"1.8", - "w_dzEtaNormMax":"1.8", - } - validationclass = PrimaryVertexValidation - def __init__(self, config): - super(PlottingOptionsPrimaryVertex, self).__init__(config, "primaryvertex") - for name in "autoLimits", "doMaps", "stdResiduals": - self.general[name] = cppboolstring(self.general[name], name) - -class PlottingOptionsOverlap(BasePlottingOptions): - validationclass = OverlapValidation - def __init__(self, config): - super(PlottingOptionsOverlap, self).__init__(config, "overlap") - -class PlottingOptionsPVResolution(BasePlottingOptions): - defaults = {} - validationclass = PrimaryVertexResolution - def __init__(self, config): - super(PlottingOptionsPVResolution, self).__init__(config, "pvresolution") - -def PlottingOptions(config, valType): - plottingOptionsClasses = { - "offline": PlottingOptionsOffline, - "split": PlottingOptionsTrackSplitting, - "zmumu": PlottingOptionsZMuMu, - "primaryvertex": PlottingOptionsPrimaryVertex, - "overlap": PlottingOptionsOverlap, - "pvresolution": PlottingOptionsPVResolution, - } - if isinstance(valType, type): - valType = valType.valType - - if valType not in globalDictionaries.plottingOptions: - if config is None: - raise ValueError("Have to provide a config the first time you call PlottingOptions for {}".format(valType)) - globalDictionaries.plottingOptions[valType] = plottingOptionsClasses[valType](config) - return globalDictionaries.plottingOptions[valType].getRepMap() - - - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/preexistingValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/preexistingValidation.py deleted file mode 100644 index 158eebb2a8db5..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/preexistingValidation.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import absolute_import -import os -from .genericValidation import GenericValidation, GenericValidationData -from .geometryComparison import GeometryComparison -from .helperFunctions import boolfromstring, getCommandOutput2, parsecolor, parsestyle -from .monteCarloValidation import MonteCarloValidation -from .offlineValidation import OfflineValidation -from .primaryVertexValidation import PrimaryVertexValidation -from .plottingOptions import PlottingOptions -from .TkAlExceptions import AllInOneError -from .trackSplittingValidation import TrackSplittingValidation -from .zMuMuValidation import ZMuMuValidation - -class PreexistingValidation(GenericValidation): - """ - Object representing a validation that has already been run, - but should be included in plots. - """ - defaults = {"title": ".oO[name]Oo."} - mandatories = {"file", "color", "style", "originalValName", "eosdirName", "multiIOV"} - removemandatories = {"dataset", "maxevents", "trackcollection"} - def __init__(self, valName, config): - self.general = config.getGeneral() - self.name = self.general["name"] = valName - self.config = config - - theUpdate = config.getResultingSection("preexisting"+self.valType+":"+self.name, - defaultDict = self.defaults, - demandPars = self.mandatories) - self.general.update(theUpdate) - - self.originalValName = self.general["originalValName"] - self.title = self.general["title"] - if "|" in self.title or "," in self.title or '"' in self.title: - msg = "The characters '|', '\"', and ',' cannot be used in the alignment title!" - raise AllInOneError(msg) - self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy") - self.jobid = self.general["jobid"] - if self.jobid: - try: #make sure it's actually a valid jobid - output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general) - if "is not found" in output: raise RuntimeError - except RuntimeError: - raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid) - - knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals - ignoreOpts = [] - config.checkInput("preexisting"+self.valType+":"+self.name, - knownSimpleOptions = knownOpts, - ignoreOptions = ignoreOpts) - self.jobmode = None - - try: #initialize plotting options for this validation type - result = PlottingOptions(self.config, self.valType) - except KeyError: - pass - - @property - def filesToCompare(self): - return {self.defaultReferenceName: self.general["file"]} - - def getRepMap(self): - #do not call super - try: - result = PlottingOptions(self.config, self.valType) - except KeyError: - result = {} - result.update(self.general) - result.update({ - "color": str(parsecolor(result["color"])), - "style": str(parsestyle(result["style"])), - }) - return result - - def createFiles(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - def createConfiguration(self, *args, **kwargs): - pass - def createScript(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - def createCrabCfg(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - -class PreexistingOfflineValidation(PreexistingValidation, OfflineValidation): - deprecateddefaults = { - "DMRMethod":"", - "DMRMinimum":"", - "DMROptions":"", - "OfflineTreeBaseDir":"", - "SurfaceShapes":"" - } - defaults = deprecateddefaults.copy() - def __init__(self, valName, config): - super(PreexistingOfflineValidation, self).__init__(valName, config) - for option in self.deprecateddefaults: - if self.general[option]: - raise AllInOneError("The '%s' option has been moved to the [plots:offline] section. Please specify it there."%option) - - def getRepMap(self): - result = super(PreexistingOfflineValidation, self).getRepMap() - result.update({ - "filetoplot": self.general["file"], - }) - return result - - def appendToMerge(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - -class PreexistingPrimaryVertexValidation(PreexistingValidation, PrimaryVertexValidation): - removemandatories = {"isda","ismc","runboundary","vertexcollection","lumilist","ptCut","etaCut","runControl","numberOfBins"} - def getRepMap(self): - result = super(PreexistingPrimaryVertexValidation, self).getRepMap() - result.update({ - "filetoplot": self.general["file"], - }) - return result - - def appendToMerge(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - -class PreexistingTrackSplittingValidation(PreexistingValidation, TrackSplittingValidation): - def appendToMerge(self, *args, **kwargs): - raise AllInOneError("Shouldn't be here...") - -class PreexistingMonteCarloValidation(PreexistingValidation): - pass - -class PreexistingZMuMuValidation(PreexistingValidation): - def __init__(self, *args, **kwargs): - raise AllInOneError("Preexisting Z->mumu validation not implemented") - #more complicated, it has multiple output files - -class PreexistingGeometryComparison(PreexistingValidation): - def __init__(self, *args, **kwargs): - raise AllInOneError("Preexisting geometry comparison not implemented") diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentation.py deleted file mode 100644 index 3d82b0ce4f667..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentation.py +++ /dev/null @@ -1,194 +0,0 @@ -from __future__ import print_function -from __future__ import absolute_import -from builtins import range -import abc -import math -import os -import re - -from .genericValidation import ValidationForPresentation, ValidationWithPlotsSummary -from .helperFunctions import recursivesubclasses -from .presentationTemplates import * -from .TkAlExceptions import AllInOneError - -# Plots related to a single validation: -class ValidationPlots(object): - def __init__(self, path): - if not os.path.isdir(path): - print("Error: Directory "+path+" not found!") - exit(1) - if not path.endswith('/'): - path += '/' - path = path.replace('\\', '/') # Beacause LaTeX has issues with '\'. - self.path = path - # List of plot files in given directory: - self.plots = [file for file in os.listdir(path) - if file.endswith('.eps')] - - @property - def validationclass(self): - possiblenames = [] - for cls in recursivesubclasses(ValidationForPresentation): - if cls.__abstractmethods__: continue - if cls.plotsdirname() == os.path.basename(os.path.realpath(self.path.rstrip("/"))): - return cls - possiblenames.append(cls.plotsdirname()) - raise AllInOneError("{} does not match any of the possible folder names:\n{}".format(self.path, ", ".join(possiblenames))) - -def validationclasses(validations): - from collections import OrderedDict - classes = [validation.validationclass for validation in validations] - #remove duplicates - http://stackoverflow.com/a/39835527/5228524 - classes = list(OrderedDict.fromkeys(classes)) - return classes - -# Layout of plots on a page: -class PageLayout(object): - def __init__(self, pattern=[], width=1, height=1): - self.pattern = [] # List of rows; row contains the order numbers - # of its plots; e.g. [[1,2,3], [4,5,6]] - self.width = width # Maximum width of one plot, - # with respect to textwidth. - self.height = height # Maximum height of one plot, - # with respect to textheight. - - # Sets variables for the given plots and returns the plots - # in an appropriate order: - def fit(self, plots): - rowlengths = [] - # First, try to place plots in a square. - nplots = sum(len(p) for p in plots) - length = int(math.ceil(math.sqrt(nplots))) - # Then, fill the square from the bottom and remove extra rows. - fullRows = int(nplots/length) - residual = nplots - length*fullRows - nrows = fullRows - if residual != 0: - rowlengths.append(residual) - nrows += 1 - for _ in range(fullRows): - rowlengths.append(length) - - # Now, fill the pattern. - self.pattern = [] - if residual == 0 and len(plots[0])%length != 0 and\ - len(plots[0])%nrows == 0: - # It's better to arrange plots in columns, not rows. - self.pattern.extend(list(range(i, i+nrows*(length-1)+1, nrows)) - for i in range(1, nrows+1)) - else: - if residual != 0: - self.pattern.append(list(range(1, 1+residual))) - self.pattern.extend(list(range(i, i+length)) for i in - range(residual+1, nplots-length+2, length)) - - self.width = 1.0/length - self.height = 0.8/nrows - - -# Write a set of pages, one for each subdetector. -# Arguments: identifier: regular expression to get the wanted plots, -# used together with subdetector name -# title: title of the plot type -# validations: list of relevant ValidationPlots objects. -# Returns the parsed script. -class SubsectionBase(object): - __metaclass__ = abc.ABCMeta - def __init__(self, title): - self.title = title - def write(self, validations): - script = '\n'.join(_ for _ in self.pages(validations) if _) - if script != '': - script = subsectionTemplate.replace('[title]', self.title)+script - return script - @abc.abstractmethod - def pages(self, validations): - pass - -class SubsectionOnePage(SubsectionBase): - def __init__(self, identifier, title): - self.identifier = identifier - super(SubsectionOnePage, self).__init__(title) - def pages(self, validations): - return [writePageReg(self.identifier, self.title, validations)] - -class SubsectionFromList(SubsectionBase): - def __init__(self, identifier, title): - self.identifier = identifier - super(SubsectionFromList, self).__init__(title) - def pages(self, validations): - return [writePageReg('(?=.*%s)%s'%(pageidentifier, self.identifier), - self.title+': ' +pagetitle, validations) - for pageidentifier, pagetitle in self.pageidentifiers] - @abc.abstractproperty - def pageidentifiers(self): - pass - -class SummarySection(SubsectionBase): - def __init__(self): - super(SummarySection, self).__init__("Summary") - def pages(self, validations): - return [summaryTemplate.replace('[title]', self.title) - .replace('[summary]', validation.validationclass.summaryitemsstring(folder=validation.path, latex=True)) - .replace("tabular", "longtable") for validation in validations - if issubclass(validation.validationclass, ValidationWithPlotsSummary)] - -# Write a page containing plots of given type. -# Arguments: identifier: regular expression to get the wanted plots -# title: title of the plot type -# validations: list of relevant ValidationPlots objects -# layout: given page layout. -# Returns the parsed script. -def writePageReg(identifier, title, validations, layout=0): - plots = [] - for validation in validations: - valiplots = [validation.path+plot for plot in validation.plots - if re.search(identifier, plot)] - valiplots.sort(key=plotSortKey) - plots.append(valiplots) - if sum(len(p) for p in plots) == 0: - print('Warning: no plots matching ' + identifier) - return '' - - # Create layout, if not given. - if layout == 0: - layout = PageLayout() - layout.fit(plots) - - return writePage([p for vali in plots for p in vali], title, layout) - - -# Write the given plots on a page. -# Arguments: plots: paths of plots to be drawn on the page -# title: title of the plot type -# layout: a PageLayout object definig the layout. -# Returns the parsed script. -def writePage(plots, title, layout): - plotrows = [] - for row in layout.pattern: - plotrow = [] - for i in range(len(row)): - plotrow.append(plotTemplate.replace('[width]', str(layout.width)).\ - replace('[height]', str(layout.height)).\ - replace('[path]', plots[row[i]-1])) - plotrows.append('\n'.join(plotrow)) - script = ' \\\\\n'.join(plotrows) - - return frameTemplate.replace('[plots]', script).replace('[title]', title) - - -# Sort key to rearrange a plot list. -# Arguments: plot: to be sorted. -def plotSortKey(plot): - # Move normchi2 before chi2Prob - if plot.find('normchi2') != -1: - return 'chi2a' - if plot.find('chi2Prob') != -1: - return 'chi2b' - return plot - -import Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison -import Alignment.OfflineValidation.TkAlAllInOneTool.offlineValidation -import Alignment.OfflineValidation.TkAlAllInOneTool.trackSplittingValidation -import Alignment.OfflineValidation.TkAlAllInOneTool.primaryVertexValidation -import Alignment.OfflineValidation.TkAlAllInOneTool.zMuMuValidation diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentationTemplates.py deleted file mode 100644 index 0566c51997b63..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/presentationTemplates.py +++ /dev/null @@ -1,95 +0,0 @@ -# Templates for the production of a LaTeX presentation. - -texTemplate=r"""%Offline Alignment Validation presentation. -%Time of creation: [time] -%Created with produceOfflineValidationTex.py -\documentclass{beamer} -\usepackage[latin1]{inputenc} -\usepackage{color} -\usepackage{longtable} -%\usepackage{siunitx} -%\usepackage{epstopdf} -\usetheme{default} -\title[Offline Validation]{Title here} -\author{Author(s) here} -\institute{Institute here} -\date{Date here} - - - -\begin{document} - -\begin{frame} -\titlepage -\end{frame} - -\section{Introduction} -%--------------------------------------------- -\begin{frame}{Introduction} -{ -\begin{itemize} - \item Introduction here -\end{itemize} -} -\end{frame} - - -\section{Plots} -%--------------------------------------------- - - -[frames] - - -\section{Conclusions} -%--------------------------------------------- -\begin{frame}{Conclusions} -{ -\begin{itemize} -\item Conclusions here -\end{itemize} -} -\end{frame} - - - -\end{document} - -""" - -frameTemplate=r""" -\begin{frame}{[title]} - \begin{figure} - \centering -[plots] - %\\Comments here - \end{figure} -\end{frame} -""" - -summaryTemplate = r""" -\begin{frame}[allowframebreaks]{[title]} -\centering -[summary] -\end{frame} -""" - -plotTemplate=r""" \includegraphics[width=[width]\textwidth, height=[height]\textheight, keepaspectratio=true]{[path]}""" - -subsectionTemplate=r""" - -\subsection{[title]} -%--------------------------------------------- -""" - -toPdf=""" -#To produce a pdf presentation -#a. fill in your information, comments etc. in presentation.tex -#b. run this script: ./ToPdf.sh -latex presentation.tex -latex presentation.tex #(twice to produce the bookmarks) -dvipdf presentation.dvi -#(pdflatex doesn't like .eps-images; this way we can -#use just latex and the convert the result into pdf.) - -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolution.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolution.py deleted file mode 100644 index 3aaeb7719055b..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolution.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -from . import configTemplates -from . import globalDictionaries -from .genericValidation import GenericValidationData, ValidationWithPlots, pythonboolstring -from .helperFunctions import replaceByMap -from .TkAlExceptions import AllInOneError - -class PrimaryVertexResolution(GenericValidationData, ValidationWithPlots): - configBaseName = "TkAlPrimaryVertexResolution" - scriptBaseName = "TkAlPrimaryVertexResolution" - crabCfgBaseName = "TkAlPrimaryVertexResolution" - resultBaseName = "PrimaryVertexResolution" - outputBaseName = "PrimaryVertexResolution" - defaults = { - # N.B.: the reference needs to be updated each time the format of the output is changed - "pvresolutionreference": ("/store/group/alca_trackeralign/validation/PVResolution/Reference/PrimaryVertexResolution_phaseIMC92X_upgrade2017_Ideal.root"), - "multiIOV":"False", - "startScale":"1.", - "endScale":"1000.", - "nTracksBins":"60.", - "nVtxBins":"40." - } - #mandatories = {"isda","ismc","runboundary","trackcollection","vertexcollection","lumilist","ptCut","etaCut","runControl","numberOfBins"} - mandatories = {"runControl","runboundary","doTriggerSelection","triggerBits","trackcollection"} - valType = "pvresolution" - def __init__(self, valName, alignment, config): - super(PrimaryVertexResolution, self).__init__(valName, alignment, config) - - if self.general["pvresolutionreference"].startswith("/store"): - self.general["pvresolutionreference"] = "root://eoscms//eos/cms" + self.general["pvresolutionreference"] - if self.NJobs > 1: - raise AllInOneError("Parallel jobs not implemented for the SplotVertexResolution validation!\n" - "Please set parallelJobs = 1.") - @property - def ValidationTemplate(self): - return configTemplates.PrimaryVertexResolutionTemplate - - @property - def TrackSelectionRefitting(self): - return configTemplates.SingleTrackRefitter - - @property - def DefinePath(self): - return configTemplates.PVResolutionPath - - @property - def ValidationSequence(self): - #never enters anywhere, since we use the custom DefinePath which includes the goodVertexSkim - return "" - - @property - def ProcessName(self): - return "PrimaryVertexResolution" - - def createScript(self, path): - return super(PrimaryVertexResolution, self).createScript(path, template = configTemplates.PVResolutionScriptTemplate) - - def createCrabCfg(self, path): - return super(PrimaryVertexResolution, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap(self, alignment = None): - if alignment == None: - alignment = self.alignmentToValidate - repMap = super(PrimaryVertexResolution, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"], - "TrackCollection": self.general["trackcollection"], - "eosdir": os.path.join(self.general["eosdir"]), - #"eosdir": os.path.join(self.general["eosdir"], "%s/%s/%s" % (self.outputBaseName, self.name, alignment.name)), - "workingdir": ".oO[datadir]Oo./%s/%s/%s" % (self.outputBaseName, self.name, alignment.name), - "plotsdir": ".oO[datadir]Oo./%s/%s/%s/plots" % (self.outputBaseName, self.name, alignment.name), - }) - - return repMap - - def appendToMerge(self): - """ - if no argument or "" is passed a string with an instantiation is returned, - else the validation is appended to the list - """ - repMap = self.getRepMap() - - parameters = " ".join(os.path.join("root://eoscms//eos/cms", file.lstrip("/")) for file in repMap["resultFiles"]) - - mergedoutputfile = os.path.join("root://eoscms//eos/cms", repMap["finalResultFile"].lstrip("/")) - return "hadd -f %s %s\n" % (mergedoutputfile, parameters) - - def appendToPlots(self): - repMap = self.getRepMap() - return (' PVResolution::loadFileList("root://eoscms//eos/cms%(finalResultFile)s",' - '"PrimaryVertexResolution","%(title)s", %(color)s, %(style)s);\n')%repMap - - @classmethod - def runPlots(cls, validations): - return configTemplates.PVResolutionPlotExecution - - @classmethod - def plottingscriptname(cls): - return "TkAlPrimaryVertexResolutionPlot.C" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.PVResolutionPlotTemplate - - @classmethod - def plotsdirname(cls): - return "PrimaryVertexResolution" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolutionTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolutionTemplates.py deleted file mode 100644 index 293ef9381a67e..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexResolutionTemplates.py +++ /dev/null @@ -1,205 +0,0 @@ -PrimaryVertexResolutionTemplate=""" - -HLTSel = .oO[doTriggerSelection]Oo. - -################################################################### -# Runs and events -################################################################### -runboundary = .oO[runboundary]Oo. -isMultipleRuns=False -if(isinstance(runboundary, (list, tuple))): - isMultipleRuns=True - print("Multiple Runs are selected") - -if(isMultipleRuns): - process.source.firstRun = cms.untracked.uint32(int(runboundary[0])) -else: - process.source.firstRun = cms.untracked.uint32(int(runboundary)) - - -################################################################### -# The trigger filter module -################################################################### -from HLTrigger.HLTfilters.triggerResultsFilter_cfi import * -process.theHLTFilter = triggerResultsFilter.clone( - triggerConditions = cms.vstring(.oO[triggerBits]Oo.), - hltResults = cms.InputTag( "TriggerResults", "", "HLT" ), - l1tResults = cms.InputTag( "" ), - throw = cms.bool(False) -) - -################################################################### -# PV refit -################################################################### -process.load("TrackingTools.TransientTrack.TransientTrackBuilder_cfi") - -from RecoVertex.PrimaryVertexProducer.OfflinePrimaryVertices_cfi import offlinePrimaryVertices -process.offlinePrimaryVerticesFromRefittedTrks = offlinePrimaryVertices.clone() -process.offlinePrimaryVerticesFromRefittedTrks.TrackLabel = cms.InputTag("TrackRefitter") -process.offlinePrimaryVerticesFromRefittedTrks.vertexCollections.maxDistanceToBeam = 1 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.maxNormalizedChi2 = 20 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.minSiliconLayersWithHits = 5 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.maxD0Significance = 5.0 -# as it was prior to https://github.com/cms-sw/cmssw/commit/c8462ae4313b6be3bbce36e45373aa6e87253c59 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.maxD0Error = 1.0 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.maxDzError = 1.0 -process.offlinePrimaryVerticesFromRefittedTrks.TkFilterParameters.minPixelLayersWithHits = 2 - -# Use compressions settings of TFile -# see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings -# settings = 100 * algorithm + level -# level is from 1 (small) to 9 (large compression) -# algo: 1 (ZLIB), 2 (LMZA) -# see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance -compressionSettings = 207 - -################################################################### -# The PV resolution module -################################################################### -process.PrimaryVertexResolution = cms.EDAnalyzer('SplitVertexResolution', - compressionSettings = cms.untracked.int32(compressionSettings), - storeNtuple = cms.bool(False), - vtxCollection = cms.InputTag("offlinePrimaryVerticesFromRefittedTrks"), - trackCollection = cms.InputTag("TrackRefitter"), - minVertexNdf = cms.untracked.double(10.), - minVertexMeanWeight = cms.untracked.double(0.5), - runControl = cms.untracked.bool(.oO[runControl]Oo.), - runControlNumber = cms.untracked.vuint32(runboundary), - sumpTStartScale = cms.untracked.double(.oO[startScale]Oo.), - sumpTEndScale = cms.untracked.double(.oO[endScale]Oo.), - nTrackBins = cms.untracked.double(.oO[nTracksBins]Oo.), - nVtxBins = cms.untracked.double(.oO[nVtxBins]Oo.) - ) -""" - -#################################################################### -#################################################################### -PVResolutionPath=""" - -process.theValidSequence = cms.Sequence(process.offlineBeamSpot + - process.TrackRefitter + - process.offlinePrimaryVerticesFromRefittedTrks + - process.PrimaryVertexResolution) -if (HLTSel): - process.p = cms.Path(process.theHLTFilter + process.theValidSequence) -else: - process.p = cms.Path(process.theValidSequence) -""" - -#################################################################### -#################################################################### -PVResolutionScriptTemplate="""#!/bin/bash -source /afs/cern.ch/cms/caf/setup.sh -export X509_USER_PROXY=.oO[scriptsdir]Oo./.user_proxy - -source /afs/cern.ch/cms/caf/setup.sh - -echo ----------------------- -echo Job started at `date` -echo ----------------------- - -export theLabel=.oO[alignmentName]Oo. -export theDate=.oO[runboundary]Oo. - -cwd=`pwd` -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scram runtime -sh` -cd $cwd - -mkdir -p .oO[datadir]Oo. -mkdir -p .oO[workingdir]Oo. -mkdir -p .oO[logdir]Oo. -rm -f .oO[logdir]Oo./*.stdout -rm -f .oO[logdir]Oo./*.stderr - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - rm -f .oO[workdir]Oo./* - cd .oO[workdir]Oo. -else - mkdir -p $cwd/TkAllInOneTool - cd $cwd/TkAllInOneTool -fi - -.oO[CommandLine]Oo. - -ls -lh . - -eos mkdir -p /store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/ - -for RootOutputFile in $(ls *root ) -do - xrdcp -f ${RootOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${RootOutputFile} - cp ${RootOutputFile} .oO[workingdir]Oo. -done - -cp .oO[Alignment/OfflineValidation]Oo./macros/FitPVResolution.C . -cp .oO[Alignment/OfflineValidation]Oo./macros/CMS_lumi.C . -cp .oO[Alignment/OfflineValidation]Oo./macros/CMS_lumi.h . - - if [[ .oO[pvresolutionreference]Oo. == *store* ]]; then xrdcp -f .oO[pvresolutionreference]Oo. PVValidation_reference.root; else ln -fs .oO[pvresolutionreference]Oo. ./PVResolution_reference.root; fi - -root -b -q "FitPVResolution.C(\\"${PWD}/${RootOutputFile}=${theLabel},${PWD}/PVValidation_reference.root=Design simulation\\",\\"$theDate\\")" - -mkdir -p .oO[plotsdir]Oo. -for PngOutputFile in $(ls *png ); do - xrdcp -f ${PngOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile} - cp ${PngOutputFile} .oO[plotsdir]Oo. -done - -for PdfOutputFile in $(ls *pdf ); do - xrdcp -f ${PdfOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile} - cp ${PdfOutputFile} .oO[plotsdir]Oo. -done - -echo ----------------------- -echo Job ended at `date` -echo ----------------------- - -""" - -###################################################################### -###################################################################### - -PVResolutionPlotExecution=""" -#make primary vertex validation plots - -cp .oO[plottingscriptpath]Oo. . -root -x -b -q .oO[plottingscriptname]Oo.++ - -for PdfOutputFile in $(ls *pdf ); do - xrdcp -f ${PdfOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile} - cp ${PdfOutputFile} .oO[datadir]Oo./.oO[PlotsDirName]Oo. -done - -for PngOutputFile in $(ls *png ); do - xrdcp -f ${PngOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile} - cp ${PngOutputFile} .oO[datadir]Oo./.oO[PlotsDirName]Oo. -done - -""" - -###################################################################### -###################################################################### - -PVResolutionPlotTemplate=""" -/**************************************** -This can be run directly in root, or you - can run ./TkAlMerge.sh in this directory -****************************************/ - -#include "Alignment/OfflineValidation/macros/FitPVResolution.C" - -void TkAlPrimaryVertexResolutionPlot() -{ - - // initialize the plot y-axis ranges - .oO[PlottingInstantiation]Oo. - FitPVResolution("",""); - -} -""" - - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidation.py deleted file mode 100644 index 5037c10cf1a6c..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidation.py +++ /dev/null @@ -1,113 +0,0 @@ -from __future__ import absolute_import -import os -from . import configTemplates -from . import globalDictionaries -from .genericValidation import GenericValidationData_CTSR, ParallelValidation, ValidationWithPlots, pythonboolstring -from .helperFunctions import replaceByMap -from .TkAlExceptions import AllInOneError - -class PrimaryVertexValidation(GenericValidationData_CTSR, ParallelValidation, ValidationWithPlots): - configBaseName = "TkAlPrimaryVertexValidation" - scriptBaseName = "TkAlPrimaryVertexValidation" - crabCfgBaseName = "TkAlPrimaryVertexValidation" - resultBaseName = "PrimaryVertexValidation" - outputBaseName = "PrimaryVertexValidation" - defaults = { - # N.B.: the reference needs to be updated each time the format of the output is changed - "pvvalidationreference": ("/store/group/alca_trackeralign/validation/PVValidation/Reference/PrimaryVertexValidation_phaseIMC92X_upgrade2017_Ideal.root"), - "doBPix":"True", - "doFPix":"True", - "forceBeamSpot":"False", - "multiIOV":"False", - } - mandatories = {"isda","ismc","runboundary","trackcollection","vertexcollection","lumilist","ptCut","etaCut","runControl","numberOfBins"} - valType = "primaryvertex" - def __init__(self, valName, alignment, config): - super(PrimaryVertexValidation, self).__init__(valName, alignment, config) - - for name in "doBPix", "doFPix", "forceBeamSpot": - self.general[name] = pythonboolstring(self.general[name], name) - - if self.general["pvvalidationreference"].startswith("/store"): - self.general["pvvalidationreference"] = "root://eoscms//eos/cms" + self.general["pvvalidationreference"] - - @property - def ValidationTemplate(self): - return configTemplates.PrimaryVertexValidationTemplate - - @property - def DefinePath(self): - return configTemplates.PVValidationPath - - @property - def ValidationSequence(self): - #never enters anywhere, since we use the custom DefinePath which includes the goodVertexSkim - return "" - - @property - def use_d0cut(self): - return False - - @property - def isPVValidation(self): - return True - - @property - def ProcessName(self): - return "PrimaryVertexValidation" - - def createScript(self, path): - return super(PrimaryVertexValidation, self).createScript(path, template = configTemplates.PVValidationScriptTemplate) - - def createCrabCfg(self, path): - return super(PrimaryVertexValidation, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap(self, alignment = None): - if alignment == None: - alignment = self.alignmentToValidate - repMap = super(PrimaryVertexValidation, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"], - "TrackCollection": self.general["trackcollection"], - "VertexCollection": self.general["vertexcollection"], - "eosdir": os.path.join(self.general["eosdir"]), - #"eosdir": os.path.join(self.general["eosdir"], "%s/%s/%s" % (self.outputBaseName, self.name, alignment.name)), - "workingdir": ".oO[datadir]Oo./%s/%s/%s" % (self.outputBaseName, self.name, alignment.name), - "plotsdir": ".oO[datadir]Oo./%s/%s/%s/plots" % (self.outputBaseName, self.name, alignment.name), - "filetoplot": "root://eoscms//eos/cms.oO[finalResultFile]Oo.", - }) - - return repMap - - def appendToMerge(self): - """ - if no argument or "" is passed a string with an instantiation is returned, - else the validation is appended to the list - """ - repMap = self.getRepMap() - - parameters = " ".join(os.path.join("root://eoscms//eos/cms", file.lstrip("/")) for file in repMap["resultFiles"]) - - mergedoutputfile = os.path.join("root://eoscms//eos/cms", repMap["finalResultFile"].lstrip("/")) - return "hadd -f %s %s\n" % (mergedoutputfile, parameters) - - def appendToPlots(self): - repMap = self.getRepMap() - return (' loadFileList("%(filetoplot)s",' - '"PVValidation", "%(title)s", %(color)s, %(style)s);\n')%repMap - - @classmethod - def runPlots(cls, validations): - return configTemplates.PrimaryVertexPlotExecution - - @classmethod - def plottingscriptname(cls): - return "TkAlPrimaryVertexValidationPlot.C" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.PrimaryVertexPlotTemplate - - @classmethod - def plotsdirname(cls): - return "PrimaryVertexValidation" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidationTemplates.py deleted file mode 100644 index 4c13a0bd56727..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/primaryVertexValidationTemplates.py +++ /dev/null @@ -1,320 +0,0 @@ -PrimaryVertexValidationTemplate=""" - -isDA = .oO[isda]Oo. -isMC = .oO[ismc]Oo. - -################################################################### -# Runs and events -################################################################### -runboundary = .oO[runboundary]Oo. -isMultipleRuns=False -if(isinstance(runboundary, (list, tuple))): - isMultipleRuns=True - print("Multiple Runs are selected") - -if(isMultipleRuns): - process.source.firstRun = cms.untracked.uint32(int(runboundary[0])) -else: - process.source.firstRun = cms.untracked.uint32(int(runboundary)) - -################################################################### -# JSON Filtering -################################################################### -if isMC: - print(">>>>>>>>>> testPVValidation_cfg.py: msg%-i: This is simulation!") - runboundary = 1 -else: - print(">>>>>>>>>> testPVValidation_cfg.py: msg%-i: This is real DATA!") - if ('.oO[lumilist]Oo.'): - print(">>>>>>>>>> testPVValidation_cfg.py: msg%-i: JSON filtering with: .oO[lumilist]Oo. ") - import FWCore.PythonUtilities.LumiList as LumiList - process.source.lumisToProcess = LumiList.LumiList(filename ='.oO[lumilist]Oo.').getVLuminosityBlockRange() - -#################################################################### -# Produce the Transient Track Record in the event -#################################################################### -process.load("TrackingTools.TransientTrack.TransientTrackBuilder_cfi") - -#################################################################### -# Load and Configure event selection -#################################################################### -process.primaryVertexFilter = cms.EDFilter("VertexSelector", - src = cms.InputTag(".oO[VertexCollection]Oo."), - cut = cms.string("!isFake && ndof > 4 && abs(z) <= 24 && position.Rho <= 2"), - filter = cms.bool(True) - ) - -process.noscraping = cms.EDFilter("FilterOutScraping", - applyfilter = cms.untracked.bool(True), - src = cms.untracked.InputTag(".oO[TrackCollection]Oo."), - debugOn = cms.untracked.bool(False), - numtrack = cms.untracked.uint32(10), - thresh = cms.untracked.double(0.25) - ) - - -process.load("Alignment.CommonAlignment.filterOutLowPt_cfi") -process.filterOutLowPt.src = ".oO[TrackCollection]Oo." -process.filterOutLowPt.ptmin = .oO[ptCut]Oo. -process.filterOutLowPt.runControl = .oO[runControl]Oo. -if(isMultipleRuns): - process.filterOutLowPt.runControlNumber.extend((runboundary)) -else: - process.filterOutLowPt.runControlNumber = [runboundary] - -if isMC: - process.goodvertexSkim = cms.Sequence(process.noscraping + process.filterOutLowPt) -else: - process.goodvertexSkim = cms.Sequence(process.primaryVertexFilter + process.noscraping + process.filterOutLowPt) - -#################################################################### -# Imports of parameters -#################################################################### -from RecoVertex.PrimaryVertexProducer.OfflinePrimaryVertices_cfi import offlinePrimaryVertices -## modify the parameters which differ -FilteringParams = offlinePrimaryVertices.TkFilterParameters.clone( - maxNormalizedChi2 = 5.0, # chi2ndof < 5 - maxD0Significance = 5.0, # fake cut (requiring 1 PXB hit) - maxEta = 5.0, # as per recommendation in PR #18330 -) - -## MM 04.05.2017 (use settings as in: https://github.com/cms-sw/cmssw/pull/18330) -from RecoVertex.PrimaryVertexProducer.TkClusParameters_cff import DA_vectParameters -DAClusterizationParams = DA_vectParameters.clone() - -GapClusterizationParams = cms.PSet(algorithm = cms.string('gap'), - TkGapClusParameters = cms.PSet(zSeparation = cms.double(0.2)) # 0.2 cm max separation betw. clusters - ) - -#################################################################### -# Deterministic annealing clustering or Gap clustering -#################################################################### -def switchClusterizerParameters(da): - if da: - print(">>>>>>>>>> testPVValidation_cfg.py: msg%-i: Running DA Algorithm!") - return DAClusterizationParams - else: - print(">>>>>>>>>> testPVValidation_cfg.py: msg%-i: Running GAP Algorithm!") - return GapClusterizationParams - -# Use compressions settings of TFile -# see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings -# settings = 100 * algorithm + level -# level is from 1 (small) to 9 (large compression) -# algo: 1 (ZLIB), 2 (LMZA) -# see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance -compressionSettings = 207 - -#################################################################### -# Configure the PVValidation Analyzer module -#################################################################### -process.PVValidation = cms.EDAnalyzer("PrimaryVertexValidation", - compressionSettings = cms.untracked.int32(compressionSettings), - TrackCollectionTag = cms.InputTag("FinalTrackRefitter"), - VertexCollectionTag = cms.InputTag(".oO[VertexCollection]Oo."), - Debug = cms.bool(False), - storeNtuple = cms.bool(False), - useTracksFromRecoVtx = cms.bool(False), - isLightNtuple = cms.bool(True), - askFirstLayerHit = cms.bool(False), - forceBeamSpot = cms.untracked.bool(.oO[forceBeamSpot]Oo.), - probePt = cms.untracked.double(.oO[ptCut]Oo.), - probeEta = cms.untracked.double(.oO[etaCut]Oo.), - doBPix = cms.untracked.bool(.oO[doBPix]Oo.), - doFPix = cms.untracked.bool(.oO[doFPix]Oo.), - numberOfBins = cms.untracked.int32(.oO[numberOfBins]Oo.), - runControl = cms.untracked.bool(.oO[runControl]Oo.), - runControlNumber = cms.untracked.vuint32(runboundary), - TkFilterParameters = FilteringParams, - TkClusParameters = switchClusterizerParameters(isDA) - ) -""" - -#################################################################### -#################################################################### -PVValidationPath=""" -process.p = cms.Path(process.goodvertexSkim* - process.seqTrackselRefit* - process.PVValidation) -""" - -#################################################################### -#################################################################### -PVValidationScriptTemplate="""#!/bin/bash -source /afs/cern.ch/cms/caf/setup.sh -export X509_USER_PROXY=.oO[scriptsdir]Oo./.user_proxy - -echo ----------------------- -echo Job started at `date` -echo ----------------------- - -export theLabel=.oO[alignmentName]Oo. -export theDate=.oO[runboundary]Oo. - -cwd=`pwd` -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scram runtime -sh` -cd $cwd - -mkdir -p .oO[datadir]Oo. -mkdir -p .oO[workingdir]Oo. -mkdir -p .oO[logdir]Oo. -rm -f .oO[logdir]Oo./*.stdout -rm -f .oO[logdir]Oo./*.stderr - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - rm -f .oO[workdir]Oo./* - cd .oO[workdir]Oo. -else - mkdir -p $cwd/TkAllInOneTool - cd $cwd/TkAllInOneTool -fi - -.oO[CommandLine]Oo. - -ls -lh . - -eos mkdir -p /store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/ -for RootOutputFile in $(ls *root ) -do - xrdcp -f ${RootOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./${RootOutputFile} - cp ${RootOutputFile} .oO[workingdir]Oo. -done - -cp .oO[Alignment/OfflineValidation]Oo./macros/FitPVResiduals.C . -cp .oO[Alignment/OfflineValidation]Oo./macros/CMS_lumi.C . -cp .oO[Alignment/OfflineValidation]Oo./macros/CMS_lumi.h . - - if [[ .oO[pvvalidationreference]Oo. == *store* ]]; then xrdcp -f .oO[pvvalidationreference]Oo. PVValidation_reference.root; else ln -fs .oO[pvvalidationreference]Oo. ./PVValidation_reference.root; fi - -echo "I am going to produce the comparison with IDEAL geometry of ${RootOutputFile}" -root -b -q "FitPVResiduals.C++g(\\"${PWD}/${RootOutputFile}=${theLabel},${PWD}/PVValidation_reference.root=Design simulation\\",true,true,\\"$theDate\\")" - -mkdir -p .oO[plotsdir]Oo. -for PngOutputFile in $(ls *png ); do - xrdcp -f ${PngOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile} - cp ${PngOutputFile} .oO[plotsdir]Oo. -done - -for PdfOutputFile in $(ls *pdf ); do - xrdcp -f ${PdfOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile} - cp ${PdfOutputFile} .oO[plotsdir]Oo. -done - -mkdir .oO[plotsdir]Oo./Biases/ -mkdir .oO[plotsdir]Oo./Biases/dzPhi -mkdir .oO[plotsdir]Oo./Biases/dxyPhi -mkdir .oO[plotsdir]Oo./Biases/dzEta -mkdir .oO[plotsdir]Oo./Biases/dxyEta -mkdir .oO[plotsdir]Oo./Fit -mkdir .oO[plotsdir]Oo./dxyVsEta -mkdir .oO[plotsdir]Oo./dzVsEta -mkdir .oO[plotsdir]Oo./dxyVsPhi -mkdir .oO[plotsdir]Oo./dzVsPhi -mkdir .oO[plotsdir]Oo./dxyVsEtaNorm -mkdir .oO[plotsdir]Oo./dzVsEtaNorm -mkdir .oO[plotsdir]Oo./dxyVsPhiNorm -mkdir .oO[plotsdir]Oo./dzVsPhiNorm - -mv .oO[plotsdir]Oo./BiasesCanvas* .oO[plotsdir]Oo./Biases/ -mv .oO[plotsdir]Oo./dzPhiBiasCanvas* .oO[plotsdir]Oo./Biases/dzPhi -mv .oO[plotsdir]Oo./dxyPhiBiasCanvas* .oO[plotsdir]Oo./Biases/dxyPhi -mv .oO[plotsdir]Oo./dzEtaBiasCanvas* .oO[plotsdir]Oo./Biases/dzEta -mv .oO[plotsdir]Oo./dxyEtaBiasCanvas* .oO[plotsdir]Oo./Biases/dxyEta -mv .oO[plotsdir]Oo./dzPhiTrendFit* .oO[plotsdir]Oo./Fit -mv .oO[plotsdir]Oo./dxyEtaTrendNorm* .oO[plotsdir]Oo./dxyVsEtaNorm -mv .oO[plotsdir]Oo./dzEtaTrendNorm* .oO[plotsdir]Oo./dzVsEtaNorm -mv .oO[plotsdir]Oo./dxyPhiTrendNorm* .oO[plotsdir]Oo./dxyVsPhiNorm -mv .oO[plotsdir]Oo./dzPhiTrendNorm* .oO[plotsdir]Oo./dzVsPhiNorm -mv .oO[plotsdir]Oo./dxyEtaTrend* .oO[plotsdir]Oo./dxyVsEta -mv .oO[plotsdir]Oo./dzEtaTrend* .oO[plotsdir]Oo./dzVsEta -mv .oO[plotsdir]Oo./dxyPhiTrend* .oO[plotsdir]Oo./dxyVsPhi -mv .oO[plotsdir]Oo./dzPhiTrend* .oO[plotsdir]Oo./dzVsPhi - -wget https://raw.githubusercontent.com/mmusich/PVToolScripts/master/PolishedScripts/index.php - -cp index.php .oO[plotsdir]Oo./Biases/ -cp index.php .oO[plotsdir]Oo./Biases/dzPhi -cp index.php .oO[plotsdir]Oo./Biases/dxyPhi -cp index.php .oO[plotsdir]Oo./Biases/dzEta -cp index.php .oO[plotsdir]Oo./Biases/dxyEta -cp index.php .oO[plotsdir]Oo./Fit -cp index.php .oO[plotsdir]Oo./dxyVsEta -cp index.php .oO[plotsdir]Oo./dzVsEta -cp index.php .oO[plotsdir]Oo./dxyVsPhi -cp index.php .oO[plotsdir]Oo./dzVsPhi -cp index.php .oO[plotsdir]Oo./dxyVsEtaNorm -cp index.php .oO[plotsdir]Oo./dzVsEtaNorm -cp index.php .oO[plotsdir]Oo./dxyVsPhiNorm -cp index.php .oO[plotsdir]Oo./dzVsPhiNorm - - -echo ----------------------- -echo Job ended at `date` -echo ----------------------- - -""" - -###################################################################### -###################################################################### - -PrimaryVertexPlotExecution=""" -#make primary vertex validation plots - -cp .oO[plottingscriptpath]Oo. . -root -x -b -q .oO[plottingscriptname]Oo.++ - -for PdfOutputFile in $(ls *pdf ); do - xrdcp -f ${PdfOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PdfOutputFile} - cp ${PdfOutputFile} .oO[datadir]Oo./.oO[PlotsDirName]Oo. -done - -for PngOutputFile in $(ls *png ); do - xrdcp -f ${PngOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/${PngOutputFile} - cp ${PngOutputFile} .oO[datadir]Oo./.oO[PlotsDirName]Oo. -done - -""" - -###################################################################### -###################################################################### - -PrimaryVertexPlotTemplate=""" -/**************************************** -This can be run directly in root, or you - can run ./TkAlMerge.sh in this directory -****************************************/ - -#include "Alignment/OfflineValidation/macros/FitPVResiduals.C" - -void TkAlPrimaryVertexValidationPlot() -{ - - // initialize the plot y-axis ranges - thePlotLimits->init(.oO[m_dxyPhiMax]Oo., // mean of dxy vs Phi - .oO[m_dzPhiMax]Oo., // mean of dz vs Phi - .oO[m_dxyEtaMax]Oo., // mean of dxy vs Eta - .oO[m_dzEtaMax]Oo., // mean of dz vs Eta - .oO[m_dxyPhiNormMax]Oo., // mean of dxy vs Phi (norm) - .oO[m_dzPhiNormMax]Oo., // mean of dz vs Phi (norm) - .oO[m_dxyEtaNormMax]Oo., // mean of dxy vs Eta (norm) - .oO[m_dzEtaNormMax]Oo., // mean of dz vs Eta (norm) - .oO[w_dxyPhiMax]Oo., // width of dxy vs Phi - .oO[w_dzPhiMax]Oo., // width of dz vs Phi - .oO[w_dxyEtaMax]Oo., // width of dxy vs Eta - .oO[w_dzEtaMax]Oo., // width of dz vs Eta - .oO[w_dxyPhiNormMax]Oo., // width of dxy vs Phi (norm) - .oO[w_dzPhiNormMax]Oo., // width of dz vs Phi (norm) - .oO[w_dxyEtaNormMax]Oo., // width of dxy vs Eta (norm) - .oO[w_dzEtaNormMax]Oo. // width of dz vs Eta (norm) - ); - - .oO[PlottingInstantiation]Oo. - FitPVResiduals("",.oO[stdResiduals]Oo.,.oO[doMaps]Oo.,"",.oO[autoLimits]Oo.); -} -""" - - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidation.py deleted file mode 100644 index b27297c90ff78..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidation.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import -import os -from . import configTemplates -from .genericValidation import GenericValidationData_CTSR, ParallelValidation, ValidationForPresentation, ValidationWithPlotsSummary -from .helperFunctions import replaceByMap -from .presentation import SubsectionFromList, SubsectionOnePage -from .TkAlExceptions import AllInOneError - - -class TrackSplittingValidation(GenericValidationData_CTSR, ParallelValidation, ValidationWithPlotsSummary, ValidationForPresentation): - configBaseName = "TkAlTrackSplitting" - scriptBaseName = "TkAlTrackSplitting" - crabCfgBaseName = "TkAlTrackSplitting" - resultBaseName = "TrackSplitting" - outputBaseName = "TrackSplitting" - defaults = {"multiIOV":"False"} - mandatories = {"trackcollection"} - valType = "split" - - @property - def ValidationTemplate(self): - return configTemplates.TrackSplittingTemplate - - @property - def ValidationSequence(self): - return configTemplates.TrackSplittingSequence - - @property - def ProcessName(self): - return "splitter" - - def createScript(self, path): - return super(TrackSplittingValidation, self).createScript(path) - - def createCrabCfg(self, path): - return super(TrackSplittingValidation, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap( self, alignment = None ): - repMap = super(TrackSplittingValidation, self).getRepMap(alignment) - if repMap["subdetector"] == "none": - subdetselection = "" - else: - subdetselection = "process.AlignmentTrackSelector.minHitsPerSubDet.in.oO[subdetector]Oo. = 2" - repMap.update({ - "nEvents": self.general["maxevents"], - "TrackCollection": self.general["trackcollection"], - "subdetselection": subdetselection, - }) - # repMap["outputFile"] = os.path.abspath( repMap["outputFile"] ) - # if self.jobmode.split( ',' )[0] == "crab": - # repMap["outputFile"] = os.path.basename( repMap["outputFile"] ) - return repMap - - def appendToPlots(self): - """ - if no argument or "" is passed a string with an instantiation is - returned, else the validation is appended to the list - """ - repMap = self.getRepMap() - comparestring = self.getCompareStrings("TrackSplittingValidation") - return ' "{},"'.format(comparestring) - - def appendToMerge(self): - repMap = self.getRepMap() - - parameters = " ".join(os.path.join("root://eoscms//eos/cms", file.lstrip("/")) for file in repMap["resultFiles"]) - - mergedoutputfile = os.path.join("root://eoscms//eos/cms", repMap["finalResultFile"].lstrip("/")) - return "hadd -f %s %s" % (mergedoutputfile, parameters) - - @classmethod - def plottingscriptname(cls): - return "TkAlTrackSplitPlot.C" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.trackSplitPlotTemplate - - @classmethod - def plotsdirname(cls): - return "TrackSplittingPlots" - - @classmethod - def presentationsubsections(cls): - return [ - SubsectionTrackSplitting('hist.*eps$', 'Track splitting'), -# Uncomment and edit to highlight one or more profiles -# SubsectionOnePage("profile.phi_org.Delta_phi.*.eps", "modulation"), - ] - -class SubsectionTrackSplitting(SubsectionFromList): - pageidentifiers = ( - ("hist[.]Delta_pt", "$p_T$"), - ("hist[.]Delta_(eta|phi)", "Angles"), - ("hist[.]Delta_d(xy|z)", "Vertex"), - ) - diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidationTemplates.py deleted file mode 100644 index e9c7e9a76cb5c..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/trackSplittingValidationTemplates.py +++ /dev/null @@ -1,152 +0,0 @@ -###################################################################### -###################################################################### -TrackSplittingTemplate=""" - -#adding this ~doubles the efficiency of selection -process.FittingSmootherRKP5.EstimateCut = -1 - -.oO[subdetselection]Oo. - -# Use compressions settings of TFile -# see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings -# settings = 100 * algorithm + level -# level is from 1 (small) to 9 (large compression) -# algo: 1 (ZLIB), 2 (LMZA) -# see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance -compressionSettings = 207 -process.cosmicValidation = cms.EDAnalyzer("CosmicSplitterValidation", - compressionSettings = cms.untracked.int32(compressionSettings), - ifSplitMuons = cms.bool(False), - ifTrackMCTruth = cms.bool(False), - checkIfGolden = cms.bool(False), - splitTracks = cms.InputTag("FinalTrackRefitter","","splitter"), - splitGlobalMuons = cms.InputTag("muons","","splitter"), - originalTracks = cms.InputTag("FirstTrackRefitter","","splitter"), - originalGlobalMuons = cms.InputTag("muons","","Rec") -) -""" - -###################################################################### -###################################################################### -TrackSplittingSequence = "process.cosmicValidation" - - -###################################################################### -###################################################################### -trackSplitPlotExecution=""" -#make track splitting plots - -cp .oO[trackSplitPlotScriptPath]Oo. . -root -x -b -q TkAlTrackSplitPlot.C++ - -""" - -###################################################################### -###################################################################### - -trackSplitPlotTemplate=""" -#include "Alignment/OfflineValidation/macros/trackSplitPlot.C" - -/**************************************** -This can be run directly in root, or you - can run ./TkAlMerge.sh in this directory -It can be run as is, or adjusted to fit - for misalignments or to only make - certain plots -****************************************/ - -/******************************** -To make ALL plots (247 in total): - leave this file as is -********************************/ - -/************************************************************************** -to make all plots involving a single x or y variable, or both: -Uncomment the line marked (B), and fill in for xvar and yvar - -Examples: - - xvar = "dxy", yvar = "ptrel" - makes plots of dxy vs Delta_pT/pT - (4 total - profile and resolution, - of Delta_pT/pT and its pull - distribution) - xvar = "all", yvar = "pt" - makes all plots involving Delta_pT - (not Delta_pT/pT) - (30 plots total: - histogram and pull distribution, and - their mean and width as a function - of the 7 x variables) - xvar = "", yvar = "all" - makes all histograms of all y variables - (including Delta_pT/pT) - (16 plots total - 8 y variables, - regular and pull histograms) -**************************************************************************/ - -/************************************************************************************** -To make a custom selection of plots: -Uncomment the lines marked (C) and this section, and fill in matrix however you want */ - -/* -Bool_t plotmatrix[xsize][ysize]; -void fillmatrix() -{ - for (int x = 0; x < xsize; x++) - for (int y = 0; y < ysize; y++) - plotmatrix[x][y] = (.............................); -} -*/ - -/* -The variables are defined in Alignment/OfflineValidation/macros/trackSplitPlot.h - as follows: -TString xvariables[xsize] = {"", "pt", "eta", "phi", "dz", "dxy", "theta", - "qoverpt"}; - -TString yvariables[ysize] = {"pt", "pt", "eta", "phi", "dz", "dxy", "theta", - "qoverpt", ""}; -Bool_t relativearray[ysize] = {true, false, false, false, false, false, false, - false, false}; -Use matrix[x][y] = true to make that plot, and false not to make it. -**************************************************************************************/ - -/************************************************************************************* -To fit for a misalignment, which can be combined with any other option: -Uncomment the line marked (A) and this section, and choose your misalignment */ - -/* -TString misalignment = "choose one"; -double *values = 0; -double *phases = 0; -//or: -// double values[number of files] = {...}; -// double phases[number of files] = {...}; -*/ - -/* -The options for misalignment are sagitta, elliptical, skew, telescope, or layerRot. -If the magnitude and phase of the misalignment are known (i.e. Monte Carlo data using - a geometry produced by the systematic misalignment tool), make values and phases into - arrays, with one entry for each file, to make a plot of the result of the fit vs. the - misalignment value. -phases must be filled in for sagitta, elliptical, and skew if values is; - for the others it has no effect -*************************************************************************************/ - -void TkAlTrackSplitPlot() -{ - TkAlStyle::legendheader = ".oO[legendheader]Oo."; - TkAlStyle::legendoptions = ".oO[legendoptions]Oo."; - TkAlStyle::set(.oO[publicationstatus]Oo., .oO[era]Oo., ".oO[customtitle]Oo.", ".oO[customrighttitle]Oo."); - outliercut = .oO[outliercut]Oo.; - //fillmatrix(); //(C) - subdetector = ".oO[subdetector]Oo."; - makePlots( -.oO[PlottingInstantiation]Oo. - , - //misalignment,values,phases, //(A) - ".oO[datadir]Oo./.oO[PlotsDirName]Oo." - //,"xvar","yvar" //(B) - //,plotmatrix //(C) - ); -} -""" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidation.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidation.py deleted file mode 100644 index 99453561480bb..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidation.py +++ /dev/null @@ -1,132 +0,0 @@ -from __future__ import absolute_import -import os -from . import configTemplates -from . import globalDictionaries -from .genericValidation import GenericValidationData, ValidationWithPlots -from .helperFunctions import replaceByMap -from .TkAlExceptions import AllInOneError - - -class ZMuMuValidation(GenericValidationData, ValidationWithPlots): - configBaseName = "TkAlZMuMuValidation" - scriptBaseName = "TkAlZMuMuValidation" - crabCfgBaseName = "TkAlZMuMuValidation" - resultBaseName = "ZMuMuValidation" - outputBaseName = "ZMuMuValidation" - defaults = { - "zmumureference": ("/store/caf/user/emiglior/Alignment/TkAlDiMuonValidation/Reference/BiasCheck_DYToMuMu_Summer12_TkAlZMuMu_IDEAL.root"), - "minpt" : "0.", - "maxpt" : "1000.", - "etamaxneg" : "2.4", - "etaminneg" : "-2.4", - "etamaxpos" : "2.4", - "etaminpos" : "-2.4", - "CustomMinY": "90.85", - "CustomMaxY": "91.4", - "multiIOV":"False", - } - deprecateddefaults = { - "resonance": "", - "switchONfit": "", - "rebinphi": "", - "rebinetadiff": "", - "rebineta": "", - "rebinpt": "", - } - defaults.update(deprecateddefaults) - needpackages = {'MuonAnalysis/MomentumScaleCalibration'} - valType = "zmumu" - def __init__(self, valName, alignment, config): - super(ZMuMuValidation, self).__init__(valName, alignment, config) - if self.general["zmumureference"].startswith("/store"): - self.general["zmumureference"] = "root://eoscms//eos/cms" + self.general["zmumureference"] - if self.NJobs > 1: - raise AllInOneError("Parallel jobs not implemented for the Z->mumu validation!\n" - "Please set parallelJobs = 1.") - for option in self.deprecateddefaults: - if self.general[option]: - raise AllInOneError("The '%s' option has been moved to the [plots:zmumu] section. Please specify it there."%option) - del self.general[option] - - @property - def filesToCompare(self): - return {self.defaultReferenceName: replaceByMap(".oO[eosdir]Oo./0_zmumuHisto.root", self.getRepMap())} - - @property - def ValidationTemplate(self): - return configTemplates.ZMuMuValidationTemplate - - @property - def ProcessName(self): - return "ONLYHISTOS" - - @property - def FileOutputTemplate(self): - return "" - - @property - def LoadBasicModules(self): - return super(ZMuMuValidation, self).LoadBasicModules + configTemplates.LoadMuonModules - - @property - def TrackSelectionRefitting(self): - return configTemplates.SingleTrackRefitter - - @property - def DefinePath(self): - return configTemplates.ZMuMuPath - - def createScript(self, path): - return super(ZMuMuValidation, self).createScript(path, template = configTemplates.zMuMuScriptTemplate) - - def createCrabCfg(self, path): - return super(ZMuMuValidation, self).createCrabCfg(path, self.crabCfgBaseName) - - def getRepMap(self, alignment = None): - if alignment == None: - alignment = self.alignmentToValidate - repMap = super(ZMuMuValidation, self).getRepMap(alignment) - repMap.update({ - "nEvents": self.general["maxevents"], - "outputFile": ("0_zmumuHisto.root" - ",genSimRecoPlots.root" - ",FitParameters.txt"), - "eosdir": os.path.join(self.general["eosdir"], "%s/%s/%s" % (self.outputBaseName, self.name, alignment.name)), - "workingdir": ".oO[datadir]Oo./%s/%s/%s" % (self.outputBaseName, self.name, alignment.name), - "plotsdir": ".oO[datadir]Oo./%s/%s/%s/plots" % (self.outputBaseName, self.name, alignment.name), - "TrackCollection": self.trackcollection, - }) - return repMap - - @property - def trackcollection(self): - from .plottingOptions import PlottingOptions - resonance = PlottingOptions(self.config, self.valType)["resonance"] - if resonance == "Z": - return 'ALCARECOTkAlZMuMu' - elif resonance == "JPsi": - return 'ALCARECOTkAlJpsiMuMu' - elif resonance in ("Y1S", "Y2S", "Y3S"): - return 'ALCARECOTkAlUpsilonMuMu' - else: - raise AllInOneError("Unknown resonance {}!".format(resonance)) - - def appendToPlots(self): - """ - if no argument or "" is passed a string with an instantiation is - returned, else the validation is appended to the list - """ - repMap = self.getRepMap() - return replaceByMap(' filenames.push_back("root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./BiasCheck.root"); titles.push_back(".oO[title]Oo."); colors.push_back(.oO[color]Oo.); linestyles.push_back(.oO[style]Oo.);\n', repMap) - - @classmethod - def plottingscriptname(cls): - return "TkAlMergeZmumuPlots.C" - - @classmethod - def plottingscripttemplate(cls): - return configTemplates.mergeZmumuPlotsTemplate - - @classmethod - def plotsdirname(cls): - return ".oO[resonance]Oo.MuMuPlots" diff --git a/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidationTemplates.py b/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidationTemplates.py deleted file mode 100644 index e1ea2121c27e4..0000000000000 --- a/Alignment/OfflineValidation/python/TkAlAllInOneTool/zMuMuValidationTemplates.py +++ /dev/null @@ -1,318 +0,0 @@ -ZMuMuValidationTemplate=""" -compressionSettings = 207 -###### MuSclFit SETTINGS ############################################## - -### MuScleFit specific configuration - -process.looper = cms.Looper( - "MuScleFit", - # Only used when reading events from a root tree - MaxEventsFromRootTree = cms.int32(-1), - - # Specify a file if you want to read events from a root tree in a local file. - # In this case the input source should be an empty source with 0 events. - - InputRootTreeFileName = cms.string(""), - - # Specify the file name where you want to save a root tree with the muon pairs. - # Leave empty if no file should be written. - - OutputRootTreeFileName = cms.string(""), - - - # Choose the kind of muons you want to run on - # ------------------------------------------- - MuonLabel = cms.InputTag("TrackRefitter"), - - - #MuonType = cms.int32(11), - MuonType = cms.int32(5), - - # This line allows to switch to PAT muons. Default is false. - # Note that the onia selection works only with onia patTuples. - PATmuons = cms.untracked.bool(False), - - # ---------------- # - # Select resonance # - # ---------------- # - # The resonances are to be specified in this order: - # Z0, Y(3S), Y(2S), Y(1S), Psi(2S), J/Psi - # ------------------------------------------------- - resfind = cms.vint32( - int(".oO[resonance]Oo." == "Z"), - int(".oO[resonance]Oo." == "Y3S"), - int(".oO[resonance]Oo." == "Y2S"), - int(".oO[resonance]Oo." == "Y1S"), - int(".oO[resonance]Oo." == "Psi2S"), - int(".oO[resonance]Oo." == "JPsi") - ), - - # Likelihood settings - # ------------------- - maxLoopNumber = cms.untracked.int32(1), - # Select which fits to do in which loop (0 = do not, 1 = do) - doResolFit = cms.vint32(0), - doScaleFit = cms.vint32(0), - doBackgroundFit = cms.vint32(0), - doCrossSectionFit = cms.vint32(0), - - # Use the probability file or not. If not it will perform a simpler selection taking the muon pair with - # invariant mass closer to the pdf value and will crash if some fit is attempted. - UseProbsFile = cms.untracked.bool(False), - - # False = use also MC information - speedup = cms.bool(True), - # Set this to false if you do not want to use simTracks. - # (Note that this is skipped anyway if speedup == True). - compareToSimTracks = cms.bool(False), - - # Output settings - # --------------- - # Use compressions settings of TFile - # see https://root.cern.ch/root/html534/TFile.html#TFile:SetCompressionSettings - # settings = 100 * algorithm + level - # level is from 1 (small) to 9 (large compression) - # algo: 1 (ZLIB), 2 (LMZA) - # see more about compression & performance: https://root.cern.ch/root/html534/guides/users-guide/InputOutput.html#compression-and-performance - - OutputFileName = cms.untracked.string("zmumuHisto.root"), - compressionSettings = cms.untracked.int32(compressionSettings), - - # BiasType=0 means no bias to muon momenta - # ---------------------------------------- - BiasType = cms.int32(0), - parBias = cms.vdouble(), - - # SmearType=0 means no smearing applied to muon momenta - # ----------------------------------------------------- - SmearType = cms.int32(0), - parSmear = cms.vdouble(), - - ### taken from J/Psi ######################### -# ResolFitType = cms.int32(14), -# parResol = cms.vdouble(0.007,0.015, -0.00077, 0.0063, 0.0018, 0.0164), -# parResolFix = cms.vint32(0, 0, 0,0, 0,0), -# parResolOrder = cms.vint32(0, 0, 0, 0, 0, 0), - ResolFitType = cms.int32(0), - parResol = cms.vdouble(0), - parResolFix = cms.vint32(0), - parResolOrder = cms.vint32(0), - - - # -------------------- # - # Scale fit parameters # - # -------------------- # - - # ----------------------------------------------------------------------------------- -# ScaleFitType = cms.int32(18), -# parScaleOrder = cms.vint32(0, 0, 0, 0), -# parScaleFix = cms.vint32(0, 0, 0, 0), -# parScale = cms.vdouble(1, 1, 1, 1), - ScaleFitType = cms.int32(0), - parScaleOrder = cms.vint32(0), - parScaleFix = cms.vint32(0), - parScale = cms.vdouble(0), - - - - # ---------------------------- # - # Cross section fit parameters # - # ---------------------------- # - # Note that the cross section fit works differently than the others, it - # fits ratios of parameters. Fix and Order should not be used as is, they - # are there mainly for compatibility. - parCrossSectionOrder = cms.vint32(0, 0, 0, 0, 0, 0), - parCrossSectionFix = cms.vint32(0, 0, 0, 0, 0, 0), - parCrossSection = cms.vdouble(1.233, 2.07, 6.33, 13.9, 2.169, 127.2), - - # ------------------------- # - # Background fit parameters # - # ------------------------- # - - # Window factors for: Z, Upsilons and (J/Psi,Psi2S) regions - LeftWindowBorder = cms.vdouble(70., 8., 1.391495), - RightWindowBorder = cms.vdouble(110., 12., 5.391495), - - # The two parameters of BgrFitType=2 are respectively: - # bgr fraction, (negative of) bgr exp. slope, bgr constant - # -------------------------------------------------------- - # The function types for resonances in a region must be the same - BgrFitType = cms.vint32(2, 2, 2), # regions - # These empty parameters should be used when there is no background - parBgr = cms.vdouble(0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0.,0., 0.,0., 0.,0.), - parBgrFix = cms.vint32(0, 0, 0, 0, 0, 0, - # The rest of the parameters is used for the resonance regions. They are automatically fixed in the code - # because they are never used to fit the background, but only after the rescaling. - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), - parBgrOrder = cms.vint32(0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), - - - # ----------------------- # - - # Set Minuit fit strategy - FitStrategy = cms.int32(1), - - - # Fit accuracy and debug parameters - StartWithSimplex = cms.bool(True), - ComputeMinosErrors = cms.bool(False), - MinimumShapePlots = cms.bool(True), - - ########## TO BE ENABLED ################################ - # Set the cuts on muons to be used in the fit - MinMuonPt = cms.untracked.double(.oO[minpt]Oo.), - MaxMuonPt = cms.untracked.double(.oO[maxpt]Oo.), - MinMuonEtaFirstRange = cms.untracked.double(.oO[etaminneg]Oo.), - MaxMuonEtaFirstRange = cms.untracked.double(.oO[etamaxneg]Oo.), - MinMuonEtaSecondRange = cms.untracked.double(.oO[etaminpos]Oo.), - MaxMuonEtaSecondRange = cms.untracked.double(.oO[etamaxpos]Oo.), - PileUpSummaryInfo = cms.untracked.InputTag("addPileupInfo"), - PrimaryVertexCollection = cms.untracked.InputTag("offlinePrimaryVertices"), - - # The following parameters can be used to filter events - TriggerResultsLabel = cms.untracked.string("TriggerResults"), - TriggerResultsProcess = cms.untracked.string("HLT"), - TriggerPath = cms.untracked.vstring(""), - # Negate the result of the trigger - NegateTrigger = cms.untracked.bool(False), - debug = cms.untracked.int32(0), -) - -""" - - -#################################################################### -#################################################################### -LoadMuonModules = """ -process.load("RecoMuon.DetLayers.muonDetLayerGeometry_cfi") -process.load("Geometry.MuonNumbering.muonNumberingInitialization_cfi") -process.load("RecoMuon.TrackingTools.MuonServiceProxy_cff") -process.load("Configuration.StandardSequences.Reconstruction_cff") -""" - - -#################################################################### -#################################################################### -ZMuMuPath = """ -process.p = cms.Path( - process.offlineBeamSpot*process.TrackRefitter - ) -""" - - -#################################################################### -#################################################################### -zMuMuScriptTemplate="""#!/bin/bash -source /afs/cern.ch/cms/caf/setup.sh -export X509_USER_PROXY=.oO[scriptsdir]Oo./.user_proxy - -echo ----------------------- -echo Job started at `date` -echo ----------------------- - -cwd=`pwd` -cd .oO[CMSSW_BASE]Oo./src -export SCRAM_ARCH=.oO[SCRAM_ARCH]Oo. -eval `scram runtime -sh` -cd $cwd - -mkdir -p .oO[datadir]Oo. -mkdir -p .oO[workingdir]Oo. -mkdir -p .oO[logdir]Oo. -rm -f .oO[logdir]Oo./*.stdout -rm -f .oO[logdir]Oo./*.stderr - -if [[ $HOSTNAME = lxplus[0-9]*[.a-z0-9]* ]] # check for interactive mode -then - mkdir -p .oO[workdir]Oo. - rm -f .oO[workdir]Oo./* - cd .oO[workdir]Oo. -else - mkdir -p $cwd/TkAllInOneTool - cd $cwd/TkAllInOneTool -fi - - -.oO[CommandLine]Oo. - -ls -lh . - -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/CompareBias.oO[resonance]Oo.Validation.cc . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/Legend.h . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/FitMassSlices.cc . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/FitSlices.cc . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/FitXslices.cc . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/FitWithRooFit.cc . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/FitMass1D.cc . - -root -q -b -l "CompareBias.oO[resonance]Oo.Validation.cc+(.oO[rebinphi]Oo., .oO[rebinetadiff]Oo., .oO[rebineta]Oo., .oO[rebinpt]Oo.)" - -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/tdrstyle.C . -cp .oO[MuonAnalysis/MomentumScaleCalibration]Oo./test/Macros/RooFit/MultiHistoOverlap_.oO[resonance]Oo..C . - -if [[ .oO[zmumureference]Oo. == *store* ]]; then xrdcp -f .oO[zmumureference]Oo. BiasCheck_Reference.root; else ln -fs .oO[zmumureference]Oo. ./BiasCheck_Reference.root; fi -root -q -b -l MultiHistoOverlap_.oO[resonance]Oo..C - -eos mkdir -p /store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/ -for RootOutputFile in $(ls *root ) -do - xrdcp -f ${RootOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./ - cp ${RootOutputFile} .oO[workingdir]Oo. -done - -mkdir -p .oO[plotsdir]Oo. -for PngOutputFile in $(ls *png ); do - xrdcp -f ${PngOutputFile} root://eoscms//eos/cms/store/group/alca_trackeralign/AlignmentValidation/.oO[eosdir]Oo./plots/ - cp ${PngOutputFile} .oO[plotsdir]Oo. -done - - -echo ----------------------- -echo Job ended at `date` -echo ----------------------- - -""" - -###################################################################### -###################################################################### - -mergeZmumuPlotsExecution=""" -#merge Z->mumu histograms - -cp .oO[mergeZmumuPlotsScriptPath]Oo. . -root -l -x -b -q TkAlMergeZmumuPlots.C++ - -""" - -###################################################################### -###################################################################### - -mergeZmumuPlotsTemplate=""" -#include "MuonAnalysis/MomentumScaleCalibration/test/Macros/RooFit/MultiHistoOverlapAll_.oO[resonance]Oo..C" -#include -#include - -template string separatebycommas(vector v){ - if (v.size()==0) return ""; - stringstream s; - s << v[0]; - for (unsigned int i = 1; i < v.size(); i++) s << "," << v[i]; - return s.str(); -} - -void TkAlMergeZmumuPlots(){ - vector filenames; vector titles; vector colors; vector linestyles; - -.oO[PlottingInstantiation]Oo. - - vector linestyles_new, markerstyles_new; - for (unsigned int j=0; j