Merge pull request #2 from cms-sw/master

update

Alignment/APEEstimation/python/conditions/init.py

@@ -0,0 +1 @@
+#Automatically created by SCRAM

Alignment/APEEstimation/test/SkimProducer/skimProducer_cfg.py

@@ -58,7 +58,7 @@
 if "iov" in  options.sample:
     isMultiIOV = True
     isData = True
-if options.sample == 'data1':
+elif options.sample == 'data1':
     isData1 = True
     isData = True
 elif options.sample == 'data2':

Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py

@@ -27,3 +27,20 @@
 mergeTemplate="hadd {path}/allData.root {inputFiles}"
 
 localSettingTemplate="cmsRun $CMSSW_BASE/src/Alignment/APEEstimation/test/cfgTemplate/apeLocalSetting_cfg.py {inputCommands}"
+
+conditionsFileHeader="""
+import FWCore.ParameterSet.Config as cms
+def applyConditions(process):
+"""
+
+conditionsTemplate="""
+    import CalibTracker.Configuration.Common.PoolDBESSource_cfi
+    process.my{record}Conditions = CalibTracker.Configuration.Common.PoolDBESSource_cfi.poolDBESSource.clone(
+    connect = cms.string('{connect}'),
+    toGet = cms.VPSet(cms.PSet(record = cms.string('{record}'),
+                            tag = cms.string('{tag}')
+                            )
+                    )
+    )
+    process.prefer_my{record}Conditions = cms.ESPrefer("PoolDBESSource", "my{record}Conditions")
+"""

Alignment/APEEstimation/test/autoSubmitter/config.ini

@@ -2,13 +2,11 @@
 [dataset:exampleDataset]
 # directory where to look, probably starting with /eos/cms/store/caf/user/<username>
 baseDirectory=/some/directory
-# without _ or .root at the end
-baseFileName=filename 
-# number of files in base directory
-nFiles=6
+# define input files, separate files with " ", use ranges for multiple files. [1-6,8,10] will be replaced by files with 1,2,3,4,5,6,8, and 10 in its place
+fileNames=filename.root otherFile_[1-6,8,10]_[1-2].root
 # optional, events per file, not total maxEvents
 maxEvents=-1
-# set to True for MC samples, False by default
+# optional, set to True for MC samples, False by default
 isMC=False
 
 #define alignments like this
@@ -24,14 +22,16 @@ alignmentName=alignmentObjectName
 baselineDir=Design
 # is this a baseline measurement? False by default
 isDesign=False
-# normally, 15 iterations are done for a measurement. If isDesign is True, 
-# this will be automatically set to 0 as only one iteration needs to be run
-# 15 by default
-maxIterations=15
+# You can add your conditions here. If a TrackerAlignmentRcd is defined, 
+# it will override the one defined by alignmentName
+condition TrackerAlignmentRcd=sqlite_file:/afs/asdf.db Alignments
+condition TrackerSurfaceDeformationRcd=sqlite_file:/afs/asdf.db Deformations
 
 # define measurements like this
 [measurements]
 # name: dataset alignment
 # unique names are important as this name will be used as a folder name
 # where all relevant files are stored
-exampleName: exampleDataset alignmentObjectName
+# firstIteration and maxIterations are optional, 0 and 15 by default. 
+# maxIterations is forced to 0 if alignmentObject has isDesign=True
+exampleName: exampleDataset alignmentObject firstIteration=0 maxIterations=15

Alignment/APEEstimation/test/batch/startSkim.py

@@ -8,6 +8,33 @@
 import argparse
 import multiprocessing as mp
 
+def replaceAllRanges(string):
+    if "[" in string and "]" in string:
+        strings = []
+        posS = string.find("[")
+        posE = string.find("]")
+        nums = string[posS+1:posE].split(",")
+        expression = string[posS:posE+1]
+
+        nums = string[string.find("[")+1:string.find("]")]
+        for interval in nums.split(","):
+            interval = interval.strip()
+            if "-" in interval:
+                lowNum = int(interval.split("-")[0])
+                upNum = int(interval.split("-")[1])
+                for i in range(lowNum, upNum+1):
+                    newstring = string[0:posS]+str(i)+string[posE+1:]
+                    newstring = replaceAllRanges(newstring)
+                    strings += newstring
+            else:
+                newstring = string[0:posS]+interval+string[posE+1:]
+                newstring = replaceAllRanges(newstring)
+                strings += newstring
+        return strings
+    else:
+        return [string,]
+
+
 def doSkim(sample):
     base = os.environ['CMSSW_BASE']    
 
@@ -19,14 +46,15 @@ def doSkim(sample):
     outFilePath = None
 
     # start cmsRun
-    proc = subprocess.Popen(toExec, stdout=subprocess.PIPE)
+    proc = subprocess.Popen(toExec, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
     def get_output(proc):
         while True:
             line = proc.stdout.readline().rstrip()
             if not line:
                 break
             yield line
+
 
     # print output in shell while program runs, also extract output filename
     try:
@@ -35,7 +63,7 @@ def get_output(proc):
                 outFileName = line.split("output name ")[1].split(".root")[0]
             if "Using output path" in line:
                 outFilePath = line.split("output path ")[1]
-            print(line)
+            print(sample+": "+line)
     except KeyboardInterrupt:
         #this way, the current file is closed and the skimming is finished in a way that the last opened file is actually usable
 
@@ -61,7 +89,7 @@ def get_output(proc):
                     try:
                         fileNo = int(fileNoString)
                         # For (most) weird naming conventions to not mess up renaming
-                        if len(fileNoString) != 3 or fileNo == 1:
+                        if len(fileNoString) != 3:
                             continue
 
                         newFileName = "%s_%d.root"%(outFileName, fileNo+1)
@@ -80,7 +108,8 @@ def get_output(proc):
         if not os.path.isdir(outFilePath):
             os.makedirs(outFilePath)
         for fi in targetFiles:
-            subprocess.call("xrdcp %s %s/"%(fi, outFilePath), shell=True)
+            if not subprocess.call("xrdcp %s %s/"%(fi, outFilePath), shell=True):
+                os.remove(fi)    
 
 def main(argv):
     if not 'CMSSW_BASE' in os.environ:
@@ -99,6 +128,13 @@ def main(argv):
         print("Usage: python startSkim.py -s <sample>")
         sys.exit()
 
+    finalSamples = []
+    for sample in args.samples:
+        parsedSamples = replaceAllRanges(sample)
+        finalSamples += parsedSamples
+
+    args.samples = finalSamples
+
     if len(args.samples) == 1 or args.consecutive:
         for sample in args.samples:
             doSkim(sample) 

Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py

@@ -19,6 +19,7 @@
 options.register('iterNumber', 0, VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.int, "Iteration number")
 options.register('lastIter', False, VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.bool, "Last iteration")
 options.register('alignRcd','', VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.string, "AlignmentRcd")
+options.register('conditions',"None", VarParsing.VarParsing.multiplicity.singleton, VarParsing.VarParsing.varType.string, "File with conditions")
 
 # get and parse the command line arguments
 options.parseArguments()   
@@ -56,6 +57,7 @@
 process.MessageLogger.categories.append('ApeEstimator')
 process.MessageLogger.categories.append('TrackRefitter')
 process.MessageLogger.categories.append('AlignmentTrackSelector')
+process.MessageLogger.cerr.threshold = 'WARNING'
 process.MessageLogger.cerr.INFO.limit = 0
 process.MessageLogger.cerr.default.limit = -1
 process.MessageLogger.cerr.SectorBuilder = cms.untracked.PSet(limit = cms.untracked.int32(-1))
@@ -66,7 +68,6 @@
 process.MessageLogger.cerr.FwkReport.reportEvery = 1000 ## really show only every 1000th
 
 
-
 ##
 ## Process options
 ##
@@ -157,13 +158,17 @@
 elif isData:
     process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:run2_data', '')
 
-
-
+if options.conditions != "None":
+    import importlib
+    mod = importlib.import_module("Alignment.APEEstimation.conditions.{}".format(options.conditions))
+    mod.applyConditions(process)
 
 ## Alignment and APE
 ##
 ## Choose Alignment (w/o touching APE)
-if options.alignRcd=='design':
+if options.alignRcd=='fromConditions':
+    pass # Alignment is read from the conditions file in this case
+elif options.alignRcd=='design':
     CondDBAlignment = CondDB.clone(connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS'))
     process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
         CondDBAlignment,
@@ -205,6 +210,26 @@
         )
     )
     process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
+
+elif options.alignRcd == 'mp2853':
+    CondDBAlignment = CondDB.clone()
+    process.myTrackerAlignment = cms.ESSource("PoolDBESSource",
+        CondDBAlignment,
+        timetype = cms.string("runnumber"),
+        toGet = cms.VPSet(
+            cms.PSet(
+                connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp2853/jobData/jobm3/alignments_MP.db'),
+                record = cms.string('TrackerAlignmentRcd'),
+                tag = cms.string('Alignments'),
+            ),
+            #~ cms.PSet(
+                #~ connect=cms.string('frontier://FrontierProd/CMS_CONDITIONS'),
+                #~ record=cms.string('SiPixelTemplateDBObjectRcd'),
+                #~ tag=cms.string('SiPixelTemplateDBObject_38T_TempForAlignmentReReco2018_v3'),
+            #~ )
+        )
+    )
+    process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment")
 
 elif options.alignRcd == 'hp1370':
     CondDBAlignment = CondDB.clone(connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HIP/xiaomeng/CMSSW_7_4_6_patch5/src/Alignment/HIPAlignmentAlgorithm/hp1370/alignments.db'))
@@ -311,3 +336,5 @@
     process.ApeEstimatorSequence
 )
 
+
+

Alignment/APEEstimation/test/cfgTemplate/apeLocalSetting_cfg.py

@@ -67,7 +67,7 @@
     statistics = cms.untracked.vstring('cout', 'alignment'),
     categories = cms.untracked.vstring('Alignment'),
     cout = cms.untracked.PSet(
-        threshold = cms.untracked.string('DEBUG'),
+        threshold = cms.untracked.string('WARNING'),
         noLineBreaks = cms.untracked.bool(True)
     ),
     alignment = cms.untracked.PSet(

Alignment/CocoaToDDL/src/CocoaUnitsTable.cc

@@ -64,14 +64,14 @@ CocoaUnitDefinition& CocoaUnitDefinition::operator=(const CocoaUnitDefinition& r
 
 ALIint CocoaUnitDefinition::operator==(const CocoaUnitDefinition& right) const
 {
-  return (this == (CocoaUnitDefinition *) &right);
+  return (this == &right);
 }
 
 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo....
 
 ALIint CocoaUnitDefinition::operator!=(const CocoaUnitDefinition &right) const
 {
-  return (this != (CocoaUnitDefinition *) &right);
+  return (this != &right);
 }
 
 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo....
@@ -297,14 +297,14 @@ CocoaUnitsCategory& CocoaUnitsCategory::operator=(const CocoaUnitsCategory& righ
 
 ALIint CocoaUnitsCategory::operator==(const CocoaUnitsCategory& right) const
 {
-  return (this == (CocoaUnitsCategory *) &right);
+  return (this == &right);
 }
 
 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo....
 
 ALIint CocoaUnitsCategory::operator!=(const CocoaUnitsCategory &right) const
 {
-  return (this != (CocoaUnitsCategory *) &right);
+  return (this != &right);
 }
 
 //....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo....

Alignment/CommonAlignment/src/Alignable.cc

@@ -11,7 +11,6 @@
 #include "Alignment/CommonAlignment/interface/Alignable.h"
 
 #include "CondFormats/Alignment/interface/AlignmentSurfaceDeformations.h"
-#include "CondFormats/Alignment/interface/AlignmentSorter.h"
 
 #include "Geometry/CommonTopologies/interface/SurfaceDeformation.h"
 
@@ -267,9 +266,7 @@ AlignmentSurfaceDeformations* Alignable::surfaceDeformations( void ) const
 
   std::vector<IdSurfaceDeformationPtrPair> result;
   surfaceDeformationIdPairs(result);
-  std::sort( result.begin(), 
- 	     result.end(), 
-	     lessIdAlignmentPair<IdSurfaceDeformationPtrPair>() );
+  std::sort( result.begin(), result.end(), [](auto& a, auto& b){return  a.first < b.first;});
 
   AlignmentSurfaceDeformations* allSurfaceDeformations = new AlignmentSurfaceDeformations();
 

Alignment/CommonAlignment/src/AlignableExtras.cc

@@ -15,7 +15,6 @@
 
 #include "CondFormats/Alignment/interface/Alignments.h"
 #include "CondFormats/Alignment/interface/AlignmentErrorsExtended.h"
-#include "CondFormats/Alignment/interface/AlignmentSorter.h"
 
 #include "Alignment/CommonAlignment/interface/AlignableExtras.h"
 
@@ -55,8 +54,8 @@ Alignments* AlignableExtras::alignments( void ) const
 	  delete tmpAlignments;
     }
 
-  std::sort( m_alignments->m_align.begin(), m_alignments->m_align.end(), 
-	     lessAlignmentDetId<AlignTransform>() );
+  // sort by rawId
+  std::sort( m_alignments->m_align.begin(), m_alignments->m_align.end());
 
   return m_alignments;
 }
@@ -76,8 +75,8 @@ AlignmentErrorsExtended* AlignableExtras::alignmentErrors( void ) const
 	  delete tmpAlignmentErrorsExtended;
     }
 
-  std::sort( m_alignmentErrors->m_alignError.begin(), m_alignmentErrors->m_alignError.end(), 
-	     lessAlignmentDetId<AlignTransformErrorExtended>() );
+  // sort by rawId
+  std::sort( m_alignmentErrors->m_alignError.begin(), m_alignmentErrors->m_alignError.end());
 
   return m_alignmentErrors;
 }

Alignment/CommonAlignment/src/AlignableModifier.cc

@@ -379,7 +379,7 @@ ::addDeformation(Alignable *alignable,
   }
 
   // auto_ptr has exception safe delete (in contrast to bare pointer)
-  const std::auto_ptr<SurfaceDeformation> surfDef
+  const std::unique_ptr<SurfaceDeformation> surfDef
     (SurfaceDeformationFactory::create(deformType, rndDeformation));
 
   alignable->addSurfaceDeformation(surfDef.get(), true); // true to propagate down

Alignment/CommonAlignment/src/AlignmentParametersData.cc

@@ -23,7 +23,7 @@ AlignmentParametersData::AlignmentParametersData( AlgebraicVector* param,
 {
   theNumSelected = std::count_if( theSelector->begin(),
 				  theSelector->end(),
-				  std::bind2nd( std::equal_to<bool>(), true ) );
+				  [](auto const &c){return c == true;});
 }
 
 
@@ -36,7 +36,7 @@ AlignmentParametersData::AlignmentParametersData( const AlgebraicVector& param,
 {
   theNumSelected = std::count_if( theSelector->begin(),
 				  theSelector->end(),
-				  std::bind2nd( std::equal_to<bool>(), true ) );
+				  [](auto const &c){return c == true;});
 }
 
 

Alignment/CommonAlignmentAlgorithm/plugins/SiPixelLorentzAngleCalibration.cc

@@ -336,9 +336,9 @@ void SiPixelLorentzAngleCalibration::endOfJob()
   }
 
   const unsigned int nonZeroParamsOrErrors =   // Any determined value?
-    count_if (parameters_.begin(), parameters_.end(), std::bind2nd(std::not_equal_to<double>(),0.))
+    count_if (parameters_.begin(), parameters_.end(), [] (auto c) { return c != 0.;})
     + count_if(paramUncertainties_.begin(), paramUncertainties_.end(),
-               std::bind2nd(std::not_equal_to<double>(), 0.));
+               [](auto c) { return c != 0.;});
 
   for (unsigned int iIOV = 0; iIOV < moduleGroupSelector_->numIovs(); ++iIOV) {
     auto firstRunOfIOV = static_cast<cond::Time_t>(moduleGroupSelector_->firstRunOfIOV(iIOV));

Alignment/CommonAlignmentAlgorithm/plugins/SiStripBackplaneCalibration.cc

@@ -347,9 +347,9 @@ void SiStripBackplaneCalibration::endOfJob()
   }
 
   const unsigned int nonZeroParamsOrErrors =   // Any determined value?
-    count_if (parameters_.begin(), parameters_.end(), std::bind2nd(std::not_equal_to<double>(),0.))
+    count_if (parameters_.begin(), parameters_.end(),[](auto c){return c != 0.;})
     + count_if(paramUncertainties_.begin(), paramUncertainties_.end(),
-               std::bind2nd(std::not_equal_to<double>(), 0.));
+              [](auto c){return c!= 0.;});
 
   for (unsigned int iIOV = 0; iIOV < moduleGroupSelector_->numIovs(); ++iIOV) {
     cond::Time_t firstRunOfIOV = moduleGroupSelector_->firstRunOfIOV(iIOV);

Alignment/CommonAlignmentAlgorithm/plugins/SiStripLorentzAngleCalibration.cc

@@ -366,9 +366,9 @@ void SiStripLorentzAngleCalibration::endOfJob()
   }
 
   const unsigned int nonZeroParamsOrErrors =   // Any determined value?
-    count_if (parameters_.begin(), parameters_.end(), std::bind2nd(std::not_equal_to<double>(),0.))
+    count_if (parameters_.begin(), parameters_.end(), [](auto c){return c!=0.;})
     + count_if(paramUncertainties_.begin(), paramUncertainties_.end(),
-               std::bind2nd(std::not_equal_to<double>(), 0.));
+               [](auto c){return c!=0.;});
 
   for (unsigned int iIOV = 0; iIOV < moduleGroupSelector_->numIovs(); ++iIOV) {
     auto firstRunOfIOV = static_cast<cond::Time_t>(moduleGroupSelector_->firstRunOfIOV(iIOV));