From 04dede8eb26ed6b7f2b326d72a7a94e4a1536921 Mon Sep 17 00:00:00 2001
From: mmusich <marco.musich@cern.ch>
Date: Fri, 28 May 2021 16:32:34 +0200
Subject: [PATCH 1/4] introduce unit test for AlCaHarvesting in
 Calibration/TkAlCaRecoProducers

---
 .../TkAlCaRecoProducers/test/BuildFile.xml    |  7 ++
 .../test/parseFwkJobReport.py                 | 74 +++++++++++++++++++
 .../test/testAlCaHarvesting.sh                | 14 ++++
 .../testCalibrationTkAlCaRecoProducers.cpp    |  3 +
 .../test/testPCLAlCaHarvesting.py             | 73 ++++++++++++++++++
 5 files changed, 171 insertions(+)
 create mode 100644 Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
 create mode 100755 Calibration/TkAlCaRecoProducers/test/testAlCaHarvesting.sh
 create mode 100644 Calibration/TkAlCaRecoProducers/test/testCalibrationTkAlCaRecoProducers.cpp
 create mode 100644 Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py

diff --git a/Calibration/TkAlCaRecoProducers/test/BuildFile.xml b/Calibration/TkAlCaRecoProducers/test/BuildFile.xml
index aa41fd324349f..e3641cbe533dc 100644
--- a/Calibration/TkAlCaRecoProducers/test/BuildFile.xml
+++ b/Calibration/TkAlCaRecoProducers/test/BuildFile.xml
@@ -1,3 +1,10 @@
+<environment>
+  <bin file="testCalibrationTkAlCaRecoProducers.cpp">
+    <flags TEST_RUNNER_ARGS=" /bin/bash Calibration/TkAlCaRecoProducers/test testAlCaHarvesting.sh"/>
+    <use name="FWCore/Utilities"/>
+  </bin>
+</environment>
+
 <use name="DQMServices/Core"/>
 <use name="FWCore/Framework"/>
 <use name="boost"/>
diff --git a/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py b/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
new file mode 100644
index 0000000000000..abcbd44488868
--- /dev/null
+++ b/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
@@ -0,0 +1,74 @@
+from __future__ import print_function
+import xml.etree.ElementTree as ET
+  
+## declare all constants here
+TARGET_LIST_OF_TAGS=['SiPixelQualityFromDbRcd_other', 'SiPixelQualityFromDbRcd_prompt', 'SiPixelQualityFromDbRcd_stuckTBM', 
+                     'SiStripApvGain_pcl', 'SiStripApvGainAAG_pcl', 'SiStripBadStrip_pcl', 'SiPixelAli_pcl']
+TARGET_DQM_FILES=1
+TARGET_DB_FILES=7
+
+def parseXML(xmlfile):
+  
+    # create element tree object
+    tree = ET.parse(xmlfile)
+  
+    # get root element
+    root = tree.getroot()
+
+    if( len(root.findall('AnalysisFile'))!=8):
+        print("ERROR: not found enough AnalysisFile entries in the FrameworkJobReport.xml")
+        return -1
+
+    listOfInputTags=[]
+
+    countDBfiles=0
+    countDQMfiles=0
+    # iterate news items
+    for item in root.findall('AnalysisFile'):
+        # iterate child elements of item
+        for child in item:
+            if(child.tag == 'FileName'):
+                if(child.text=='sqlite_file:promptCalibConditions.db'):
+                    countDBfiles+=1
+                elif(child.text=='./DQM_V0001_R000325022__Express__PCLTest__ALCAPROMPT.root'):
+                    countDQMfiles+=1
+                else:
+                    pass
+            if(child.tag == 'inputtag'):
+                listOfInputTags.append(child.attrib['Value'])
+
+    if(countDBfiles!=TARGET_DB_FILES):
+        print("ERROR! Found a not expected number DB files,",countDBfiles)
+        return -1
+
+    if(countDQMfiles!=TARGET_DQM_FILES):
+        print("ERROR! Found a not expected number of DQM files",countDQMfiles)
+        return -1
+
+    ## That's strict! 
+    if(listOfInputTags!=TARGET_LIST_OF_TAGS):
+        print("ERROR! This ",[x for x in listOfTags if x not in listOfInputTags]," is the set of different tags")
+        return -1
+    
+    return 0
+
+def main():
+    try:
+        f = open("FrameworkJobReport.xml")
+        # Do something with the file
+    except IOError:
+        print("File not accessible")
+
+    # parse xml file
+    result = parseXML('FrameworkJobReport.xml')
+    if(result==0):
+        print("All is fine with the world!")
+    else:
+        print("Parsing the FwkJobReport results in failure!")
+
+    return result
+
+if __name__ == "__main__":
+  
+    # calling main function
+    main()
diff --git a/Calibration/TkAlCaRecoProducers/test/testAlCaHarvesting.sh b/Calibration/TkAlCaRecoProducers/test/testAlCaHarvesting.sh
new file mode 100755
index 0000000000000..ec219c1f42858
--- /dev/null
+++ b/Calibration/TkAlCaRecoProducers/test/testAlCaHarvesting.sh
@@ -0,0 +1,14 @@
+#! /bin/bash
+
+function die { echo $1: status $2 ; exit $2; }
+function cleanTheHouse {
+    rm -fr millepede.*
+    rm -fr pede*
+    rm -fr treeFile.root
+}
+
+echo "TESTING Calibration/TkAlCaRecoProducers ..."
+cmsRun -e ${LOCAL_TEST_DIR}/testPCLAlCaHarvesting.py || die "Failure running testPCLAlCaHarvesting.py" $? 
+cleanTheHouse
+echo "PARSING Framework Job Report ..."
+python ${LOCAL_TEST_DIR}/parseFwkJobReport.py
diff --git a/Calibration/TkAlCaRecoProducers/test/testCalibrationTkAlCaRecoProducers.cpp b/Calibration/TkAlCaRecoProducers/test/testCalibrationTkAlCaRecoProducers.cpp
new file mode 100644
index 0000000000000..b2991bd18ae57
--- /dev/null
+++ b/Calibration/TkAlCaRecoProducers/test/testCalibrationTkAlCaRecoProducers.cpp
@@ -0,0 +1,3 @@
+#include "FWCore/Utilities/interface/TestHelper.h"
+
+RUNTEST()
diff --git a/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py b/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py
new file mode 100644
index 0000000000000..3d146a95c7fdb
--- /dev/null
+++ b/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py
@@ -0,0 +1,73 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process('ALCAHARVEST')
+
+# import of standard configurations
+process.load('Configuration.StandardSequences.Services_cff')
+process.load('SimGeneral.HepPDTESSource.pythiapdt_cfi')
+process.load('FWCore.MessageService.MessageLogger_cfi')
+process.load('Configuration.EventContent.EventContent_cff')
+process.load('Configuration.StandardSequences.GeometryRecoDB_cff')
+process.load('Configuration.StandardSequences.MagneticField_cff')
+process.load('Configuration.StandardSequences.AlCaHarvesting_cff')
+process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
+
+process.source = cms.Source("EmptySource",
+                            firstRun = cms.untracked.uint32(325022),
+                            numberEventsInRun = cms.untracked.uint32(1),
+                            numberEventsInLuminosityBlock = cms.untracked.uint32(1),
+                            firstTime = cms.untracked.uint64(6614916085915320320),
+                            timeBetweenEvents = cms.untracked.uint64(1)
+                            )
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(1)
+)
+
+process.PoolDBOutputService.toPut.append(process.ALCAHARVESTSiStripQuality_dbOutput)
+process.PoolDBOutputService.toPut.append(process.ALCAHARVESTSiStripGains_dbOutput)
+process.PoolDBOutputService.toPut.append(process.ALCAHARVESTSiStripGainsAAG_dbOutput )
+process.PoolDBOutputService.toPut.append(process.ALCAHARVESTSiPixelAli_dbOutput)
+process.PoolDBOutputService.toPut.extend(process.ALCAHARVESTSiPixelQuality_dbOutput)
+
+process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVESTSiStripQuality_metadata)
+process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVESTSiStripGains_metadata )
+process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVESTSiStripGainsAAG_metadata)
+process.pclMetadataWriter.recordsToMap.append(process.ALCAHARVESTSiPixelAli_metadata)
+process.pclMetadataWriter.recordsToMap.extend(process.ALCAHARVESTSiPixelQuality_metadata)
+
+process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
+from Configuration.AlCa.GlobalTag import GlobalTag
+process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:run2_data', '')
+
+process.SiStripQuality  = cms.Path(process.ALCAHARVESTSiStripQuality)
+process.alcaSiStripQualityHarvester.CalibrationThreshold = cms.untracked.uint32(0)
+
+process.SiStripGains    = cms.Path(process.ALCAHARVESTSiStripGains)
+#process.alcaSiStripGainsHarvester.
+
+process.SiStripGainsAAG = cms.Path(process.ALCAHARVESTSiStripGainsAAG)
+#process.alcaSiStripGainsAAGHarvester.
+
+process.SiPixelAli      = cms.Path(process.ALCAHARVESTSiPixelAli)
+
+process.SiPixelQuality  = cms.Path(process.ALCAHARVESTSiPixelQuality)
+
+process.ALCAHARVESTDQMSaveAndMetadataWriter = cms.Path(process.dqmSaver+process.pclMetadataWriter)
+
+process.schedule = cms.Schedule(process.SiStripQuality,
+                                process.SiStripGains,    
+                                process.SiStripGainsAAG, 
+                                process.SiPixelAli,      
+                                process.SiPixelQuality,
+                                process.ALCAHARVESTDQMSaveAndMetadataWriter)
+
+from PhysicsTools.PatAlgos.tools.helpers import associatePatAlgosToolsTask
+associatePatAlgosToolsTask(process)
+
+# Customisation from command line
+
+# Add early deletion of temporary data products to reduce peak memory need
+from Configuration.StandardSequences.earlyDeleteSettings_cff import customiseEarlyDelete
+process = customiseEarlyDelete(process)
+# End adding early deletion

From cc9852d212e9bb8cbd0cb4567dd8d6f89feb24a6 Mon Sep 17 00:00:00 2001
From: mmusich <marco.musich@cern.ch>
Date: Fri, 28 May 2021 16:40:17 +0200
Subject: [PATCH 2/4] protect the case in which there is no input histogram to
 harvest

---
 .../SiStripChannelGain/src/SiStripGainsPCLHarvester.cc         | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CalibTracker/SiStripChannelGain/src/SiStripGainsPCLHarvester.cc b/CalibTracker/SiStripChannelGain/src/SiStripGainsPCLHarvester.cc
index 6537b4e41930f..9d83de340bd53 100644
--- a/CalibTracker/SiStripChannelGain/src/SiStripGainsPCLHarvester.cc
+++ b/CalibTracker/SiStripChannelGain/src/SiStripGainsPCLHarvester.cc
@@ -258,6 +258,9 @@ void SiStripGainsPCLHarvester::gainQualityMonitor(DQMStore::IBooker& ibooker_,
 
     if (Gain != 1.) {
       std::vector<MonitorElement*> charge_histos = APVGain::FetchMonitor(new_charge_histos, DetId, tTopo_);
+
+      if (!Charge_Vs_Index)
+        continue;
       TH2S* chvsidx = (Charge_Vs_Index)->getTH2S();
       int bin = chvsidx->GetXaxis()->FindBin(Index);
       TH1D* Proj = chvsidx->ProjectionY("proj", bin, bin);

From 0a5f3d1989dcc583ea1f7750eb6b23afff79feeb Mon Sep 17 00:00:00 2001
From: mmusich <marco.musich@cern.ch>
Date: Fri, 28 May 2021 16:40:32 +0200
Subject: [PATCH 3/4] fix typo

---
 .../plugins/SiStripQualityHotStripIdentifierRoot.cc             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/DQMOffline/CalibTracker/plugins/SiStripQualityHotStripIdentifierRoot.cc b/DQMOffline/CalibTracker/plugins/SiStripQualityHotStripIdentifierRoot.cc
index 4f8a380293e9d..95906487c10de 100644
--- a/DQMOffline/CalibTracker/plugins/SiStripQualityHotStripIdentifierRoot.cc
+++ b/DQMOffline/CalibTracker/plugins/SiStripQualityHotStripIdentifierRoot.cc
@@ -290,7 +290,7 @@ void SiStripQualityHotStripIdentifierRoot::bookHistos() {
   }
   if (!gotNentries) {
     edm::LogWarning("SiStripQualityHotStripIdentifierRoot")
-        << " [SiStripQualityHotStripIdentifierRoot::bookHistos] :: Histogram with to check # of evemnts missing"
+        << " [SiStripQualityHotStripIdentifierRoot::bookHistos] :: Histogram with to check # of events missing"
         << std::endl;
   }
   for (; iter != iterEnd; ++iter) {

From 61f3cb073cfaae2f3f5a98fb8bf3e80a5df63b16 Mon Sep 17 00:00:00 2001
From: mmusich <marco.musich@cern.ch>
Date: Fri, 28 May 2021 21:19:19 +0200
Subject: [PATCH 4/4] improve scripts and actually return failure if does not
 pass checks

---
 .../test/parseFwkJobReport.py                 | 31 ++++++----
 .../test/testPCLAlCaHarvesting.py             | 58 +++++++++++++++++--
 2 files changed, 75 insertions(+), 14 deletions(-)

diff --git a/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py b/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
index abcbd44488868..6acd54e05cc61 100644
--- a/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
+++ b/Calibration/TkAlCaRecoProducers/test/parseFwkJobReport.py
@@ -1,12 +1,18 @@
 from __future__ import print_function
 import xml.etree.ElementTree as ET
-  
+import sys
+
 ## declare all constants here
 TARGET_LIST_OF_TAGS=['SiPixelQualityFromDbRcd_other', 'SiPixelQualityFromDbRcd_prompt', 'SiPixelQualityFromDbRcd_stuckTBM', 
-                     'SiStripApvGain_pcl', 'SiStripApvGainAAG_pcl', 'SiStripBadStrip_pcl', 'SiPixelAli_pcl']
+                     'SiStripApvGain_pcl', 'SiStripApvGainAAG_pcl',
+                     'SiStripBadStrip_pcl', 'SiPixelAli_pcl']
 TARGET_DQM_FILES=1
+TARGET_DQM_FILENAME='./DQM_V0001_R000325022__Express__PCLTest__ALCAPROMPT.root'
 TARGET_DB_FILES=7
+TARGET_DB_FILENAME='sqlite_file:promptCalibConditions.db'
+TOTAL_TARGET_FILES=TARGET_DQM_FILES+TARGET_DB_FILES
 
+#_____________________________________________________
 def parseXML(xmlfile):
   
     # create element tree object
@@ -15,22 +21,25 @@ def parseXML(xmlfile):
     # get root element
     root = tree.getroot()
 
-    if( len(root.findall('AnalysisFile'))!=8):
-        print("ERROR: not found enough AnalysisFile entries in the FrameworkJobReport.xml")
+    totAnaEntries=len(root.findall('AnalysisFile'))
+
+    if(totAnaEntries!=TOTAL_TARGET_FILES):
+        print("ERROR: found a not expected number (",totAnaEntries,") of AnalysisFile entries in the FrameworkJobReport.xml")
         return -1
 
     listOfInputTags=[]
 
     countDBfiles=0
     countDQMfiles=0
+
     # iterate news items
     for item in root.findall('AnalysisFile'):
         # iterate child elements of item
         for child in item:
             if(child.tag == 'FileName'):
-                if(child.text=='sqlite_file:promptCalibConditions.db'):
+                if(child.text==TARGET_DB_FILENAME):
                     countDBfiles+=1
-                elif(child.text=='./DQM_V0001_R000325022__Express__PCLTest__ALCAPROMPT.root'):
+                elif(child.text==TARGET_DQM_FILENAME):
                     countDQMfiles+=1
                 else:
                     pass
@@ -38,7 +47,7 @@ def parseXML(xmlfile):
                 listOfInputTags.append(child.attrib['Value'])
 
     if(countDBfiles!=TARGET_DB_FILES):
-        print("ERROR! Found a not expected number DB files,",countDBfiles)
+        print("ERROR! Found a not expected number of DB files",countDBfiles)
         return -1
 
     if(countDQMfiles!=TARGET_DQM_FILES):
@@ -52,22 +61,24 @@ def parseXML(xmlfile):
     
     return 0
 
+#_____________________________________________________
 def main():
     try:
         f = open("FrameworkJobReport.xml")
-        # Do something with the file
     except IOError:
         print("File not accessible")
+        sys.exit(1)
 
     # parse xml file
     result = parseXML('FrameworkJobReport.xml')
     if(result==0):
         print("All is fine with the world!")
+        sys.exit(0)
     else:
         print("Parsing the FwkJobReport results in failure!")
+        sys.exit(1)
 
-    return result
-
+#_____________________________________________________
 if __name__ == "__main__":
   
     # calling main function
diff --git a/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py b/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py
index 3d146a95c7fdb..6f9e7a31e1b96 100644
--- a/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py
+++ b/Calibration/TkAlCaRecoProducers/test/testPCLAlCaHarvesting.py
@@ -1,5 +1,52 @@
-import FWCore.ParameterSet.Config as cms
+from __future__ import print_function
+import calendar
+import CondCore.Utilities.conddblib as conddb
+
+#___________________________________________________________________
+def findRunStopTime(run_number):
+    con = conddb.connect(url = conddb.make_url("pro"))
+    session = con.session()
+    RunInfo = session.get_dbtype(conddb.RunInfo)
+    bestRun = session.query(RunInfo.run_number,RunInfo.start_time, RunInfo.end_time).filter(RunInfo.run_number >= run_number).first()
+    if bestRun is None:
+        raise Exception("Run %s can't be matched with an existing run in the database." % run_number)
+
+    start= bestRun[1]
+    stop = bestRun[2]
+
+    bestRunStartTime = calendar.timegm( bestRun[1].utctimetuple() ) << 32
+    bestRunStopTime  = calendar.timegm( bestRun[2].utctimetuple() ) << 32
+
+    print("run start time:",start,"(",bestRunStartTime,")")
+    print("run stop time: ",stop,"(",bestRunStopTime,")")
+
+    return bestRunStopTime
+
+import optparse
+parser = optparse.OptionParser(usage = 'Usage: %prog [options] <file> [<file> ...]\n')
+parser.add_option('-G', '--inputGT',
+                  dest = 'inputGT',
+                  default = 'auto:run2_data',
+                  help = 'Global Tag to get conditions')
+
+parser.add_option('-r', '--inputRun',
+                  dest = 'inputRun',
+                  default = 325022,
+                  help = 'run to be used')
+
+parser.add_option('-t', '--inputTime',
+                  dest = 'inputTime',
+                  default = 6614916085915320320,
+                  help = 'time to be used')
+
+parser.add_option('-e', '--enableJobReport',
+                  dest = 'empty',
+                  default = None,
+                  help = 'unused')
+
+(options, arguments) = parser.parse_args()
 
+import FWCore.ParameterSet.Config as cms
 process = cms.Process('ALCAHARVEST')
 
 # import of standard configurations
@@ -12,11 +59,14 @@
 process.load('Configuration.StandardSequences.AlCaHarvesting_cff')
 process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
 
+##
+## configure the source with an random run
+##
 process.source = cms.Source("EmptySource",
-                            firstRun = cms.untracked.uint32(325022),
+                            firstRun = cms.untracked.uint32(options.inputRun),
                             numberEventsInRun = cms.untracked.uint32(1),
                             numberEventsInLuminosityBlock = cms.untracked.uint32(1),
-                            firstTime = cms.untracked.uint64(6614916085915320320),
+                            firstTime = cms.untracked.uint64(options.inputTime),
                             timeBetweenEvents = cms.untracked.uint64(1)
                             )
 
@@ -38,7 +88,7 @@
 
 process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff')
 from Configuration.AlCa.GlobalTag import GlobalTag
-process.GlobalTag = GlobalTag(process.GlobalTag, 'auto:run2_data', '')
+process.GlobalTag = GlobalTag(process.GlobalTag, options.inputGT, '')
 
 process.SiStripQuality  = cms.Path(process.ALCAHARVESTSiStripQuality)
 process.alcaSiStripQualityHarvester.CalibrationThreshold = cms.untracked.uint32(0)