[14.0.X] TkAlignment: Multi-IOV Zmumu mode, fixes to DMR averaged, PV trends

Alignment/OfflineValidation/bin/Zmumumerge.cc

@@ -374,7 +374,18 @@ int Zmumumerge(int argc, char* argv[]) {
   pt::ptree alignments = main_tree.get_child("alignments");
   pt::ptree validation = main_tree.get_child("validation");
 
+  //Load defined order
+  std::vector<std::pair<std::string, pt::ptree>> alignmentsOrdered;
   for (const auto& childTree : alignments) {
+    alignmentsOrdered.push_back(childTree);
+  }
+  std::sort(alignmentsOrdered.begin(),
+            alignmentsOrdered.end(),
+            [](const std::pair<std::string, pt::ptree>& left, const std::pair<std::string, pt::ptree>& right) {
+              return left.second.get<int>("index") < right.second.get<int>("index");
+            });
+
+  for (const auto& childTree : alignmentsOrdered) {
     // do not consider the nodes with a "file" to merge
     if (childTree.second.find("file") == childTree.second.not_found()) {
       std::cerr << "Ignoring alignment: " << childTree.second.get<std::string>("title") << ".\nNo file to merged found!"

Alignment/OfflineValidation/python/TkAlAllInOneTool/DMRplotter.py

@@ -51,6 +51,17 @@ def __log__(self,log_type="",text=""):
             else:
                 print(text)
 
+    def _middleString(self, fullString):
+        ##############################################################
+        #Auxiliary function to retrieve object name from full string
+        ##############################################################
+
+        middleString = "_".join(fullString.split("_")[1:])
+        if middleString.endswith("_y"): middleString = "_".join(middleString.split("_")[:-1])
+        middleString = "_".join(middleString.split("_")[:-1])
+        return middleString
+
+
     def _replaceMulti(self, mainString, toBeReplaced, newString):
         #################################
         #Auxiliary function to remove 
@@ -339,7 +350,7 @@ def __defineObjects__(self):
                 if objName in objAreIgnored: continue   
                 objDict[objName] = []
                 for obj in objList:
-                    if objName in obj.GetName():
+                    if objName == self._middleString(obj.GetName()):
                         segment = ""
                         var = ""
                         if obj.GetName()[-1] == "y":
@@ -834,7 +845,7 @@ def addMC(self,filename):
 
     def plotSingle(self):
         ##############################################
-	#Auxiliary plotter for unweighted Data and MC
+	    #Auxiliary plotter for unweighted Data and MC
         ##############################################
 
         #check for input file and create output dir
@@ -993,7 +1004,7 @@ def plot(self):
                 #order plots & prepare y-axis scale factors  
                 isEmpty = True
                 maxY = 0.0
-                objGroup = [] 
+                objGroup = []
                 for objName in self.objNameList: #follow plotting order
                     for obj in objects[objName]:
                         if obj['var'] == var and obj['segment'] == segment:
@@ -1010,14 +1021,13 @@ def plot(self):
                                 legStyle = "l"
                             if obj['type'] == "DATA": 
                                 drawStyle += "P HIST SAME"
-                                legStyle = "p"     
+                                legStyle = "p"    
                             objGroup.append({'hist'        : obj['hist'], 
                                              'label'       : legendLabel,
                                              'stat'        : self.__getStat__(obj['hist'],var),
                                              'drawStyle'   : drawStyle,
                                              'legStyle'    : legStyle  
                                             })
-
                 #draw & save
                 if not isEmpty:
                     legMinY = (1./self.legendOffset)+(1.-1./self.legendOffset)*(self.maxEntriesPerColumn-len(objGroup))/(self.maxEntriesPerColumn*3)

Alignment/OfflineValidation/python/TkAlAllInOneTool/Zmumu.py

@@ -3,39 +3,53 @@
 
 def Zmumu(config, validationDir):
     ##List with all jobs
-    jobs = []
+    jobs, singleJobs = [], []
     zmumuType = "single"
 
-    ##List with all wished IOVs
-    IOVs = []
+    ##Dictionary of lists of all IOVs (can be different per each single job)
+    IOVs = {}
+
+    ##Auxilliary dictionary of isData flags per each merged job
+    isDataMerged = {}
 
     ##Start with single Zmumu jobs
     if not zmumuType in config["validations"]["Zmumu"]: 
         raise Exception("No 'single' key word in config for Zmumu") 
 
-    for datasetName in config["validations"]["Zmumu"][zmumuType]:
-        for IOV in config["validations"]["Zmumu"][zmumuType][datasetName]["IOV"]:
+    for singleName in config["validations"]["Zmumu"][zmumuType]:
+        aux_IOV = config["validations"]["Zmumu"][zmumuType][singleName]["IOV"]
+        if not isinstance(aux_IOV, list) and aux_IOV.endswith(".txt"):
+            config["validations"]["Zmumu"][zmumuType][singleName]["IOV"] = []
+            with open(aux_IOV, 'r') as IOVfile:
+                for line in IOVfile.readlines():
+                    if len(line) != 0: config["validations"]["Zmumu"][zmumuType][singleName]["IOV"].append(int(line))
+        for IOV in config["validations"]["Zmumu"][zmumuType][singleName]["IOV"]:
             ##Save IOV to loop later for merge jobs
-            if not IOV in IOVs:
-                IOVs.append(IOV)
+            if singleName not in IOVs.keys():
+                IOVs[singleName] = []
+            if IOV not in IOVs[singleName]:
+                IOVs[singleName].append(IOV)
 
-            for alignment in config["validations"]["Zmumu"][zmumuType][datasetName]["alignments"]:
+            for alignment in config["validations"]["Zmumu"][zmumuType][singleName]["alignments"]:
                 ##Work directory for each IOV
-                workDir = "{}/Zmumu/{}/{}/{}/{}".format(validationDir, zmumuType, datasetName, alignment, IOV)
+                workDir = "{}/Zmumu/{}/{}/{}/{}".format(validationDir, zmumuType, singleName, alignment, IOV)
 
                 ##Write local config
                 local = {}
-                local["output"] = "{}/{}/{}/{}/{}/{}".format(config["LFS"], config["name"], zmumuType, alignment, datasetName, IOV)
+                local["output"] = "{}/{}/Zmumu/{}/{}/{}/{}".format(config["LFS"], config["name"], zmumuType, alignment, singleName, IOV)
                 local["alignment"] = copy.deepcopy(config["alignments"][alignment])
-                local["validation"] = copy.deepcopy(config["validations"]["Zmumu"][zmumuType][datasetName])
+                local["alignment"]["name"] = alignment
+                local["validation"] = copy.deepcopy(config["validations"]["Zmumu"][zmumuType][singleName])
                 local["validation"].pop("alignments")
                 local["validation"]["IOV"] = IOV
+                if "dataset" in local["validation"]:
+                    local["validation"]["dataset"] = local["validation"]["dataset"].format(IOV)
                 if "goodlumi" in local["validation"]:
                     local["validation"]["goodlumi"] = local["validation"]["goodlumi"].format(IOV)
 
                 ##Write job info
                 job = {
-                    "name": "Zmumu_{}_{}_{}_{}".format(zmumuType, alignment, datasetName, IOV),
+                    "name": "Zmumu_{}_{}_{}_{}".format(zmumuType, alignment, singleName, IOV),
                     "dir": workDir,
                     "exe": "cmsRun",
                     "cms-config": "{}/src/Alignment/OfflineValidation/python/TkAlAllInOneTool/Zmumu_cfg.py".format(os.environ["CMSSW_BASE"]),
@@ -44,7 +58,9 @@ def Zmumu(config, validationDir):
                     "config": local, 
                 }
 
-                jobs.append(job)
+                singleJobs.append(job)
+
+    jobs.extend(singleJobs)            
 
     ##Do merge Zmumu if wished
     if "merge" in config["validations"]["Zmumu"]:
@@ -54,40 +70,84 @@ def Zmumu(config, validationDir):
 
         ##Loop over all merge jobs/IOVs which are wished
         for mergeName in config["validations"]["Zmumu"][zmumuType]:
-            for IOV in IOVs:
-                ##Work directory for each IOV
-                workDir = "{}/Zmumu/{}/{}/{}".format(validationDir, zmumuType, mergeName, IOV)
-
-                ##Write job info
-                local = {}
-
-                job = {
-                    "name": "Zmumu_{}_{}_{}".format(zmumuType, mergeName, IOV),
-                    "dir": workDir,
-                    "exe": "Zmumumerge",
-                    "run-mode": "Condor",
-                    "dependencies": [],
-                    "config": local, 
-                }
-
-                for alignment in config["alignments"]:
-                    ##Deep copy necessary things from global config
-                    local.setdefault("alignments", {})
-                    local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
+            ##Search for MC single(s)
+            singlesMC = []
+            for singleName in config["validations"]["Zmumu"][zmumuType][mergeName]['singles']:
+                if len(IOVs[singleName]) == 1 and int(IOVs[singleName][0]) == 1: singlesMC.append(singleName)
+            isMConly = (len(singlesMC) == len(config["validations"]["Zmumu"][zmumuType][mergeName]['singles']))
+            if isMConly:
+                isDataMerged[mergeName] = 0
+            elif len(singlesMC) == 0:
+                isDataMerged[mergeName] = 1
+            else:
+                isDataMerged[mergeName] = -1
+
+            ##Loop over singles
+            for iname,singleName in enumerate(config["validations"]["Zmumu"][zmumuType][mergeName]['singles']):
+                isMC = (singleName in singlesMC)
+                if isMConly and iname > 0: continue #special case for MC only comparison
+                elif isMConly: singlesMC.pop(singlesMC.index(singleName))
+
+                for IOV in IOVs[singleName]:
+                    ##Work directory for each IOV
+                    workDir = "{}/Zmumu/{}/{}/{}".format(validationDir, zmumuType, mergeName, IOV)
+
+                    ##Write job info
+                    local = {}
+
+                    job = {
+                        "name": "Zmumu_{}_{}_{}".format(zmumuType, mergeName, IOV),
+                        "dir": workDir,
+                        "exe": "Zmumumerge",
+                        "run-mode": "Condor",
+                        "dependencies": [],
+                        "config": local,
+                    }
+
+                    ##Deep copy necessary things from global config + assure plot order
+                    for alignment in config["alignments"]:
+                        idxIncrement = 0
+                        local.setdefault("alignments", {})
+                        if alignment in config["validations"]["Zmumu"]["single"][singleName]["alignments"]: #Cover all DATA validations
+                            local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
+                            local["alignments"][alignment]['index'] = config["validations"]["Zmumu"]["single"][singleName]["alignments"].index(alignment)
+                            local["alignments"][alignment]['isMC'] = False
+                        for singleMCname in singlesMC:
+                            if alignment in config["validations"]["Zmumu"]["single"][singleMCname]["alignments"]: #Add MC objects
+                                local["alignments"][alignment] = copy.deepcopy(config["alignments"][alignment])
+                                local["alignments"][alignment]['index']  = len(config["validations"]["Zmumu"]["single"][singleName]["alignments"])
+                                local["alignments"][alignment]['index'] += idxIncrement + config["validations"]["Zmumu"]["single"][singleMCname]["alignments"].index(alignment)
+                                local["alignments"][alignment]['isMC'] = True
+                            idxIncrement += len(config["validations"]["Zmumu"]["single"][singleMCname]["alignments"])    
                     local["validation"] = copy.deepcopy(config["validations"]["Zmumu"][zmumuType][mergeName])
-                    local["output"] = "{}/{}/{}/{}/{}".format(config["LFS"], config["name"], zmumuType, mergeName, IOV)
-
-                ##Loop over all single jobs
-                for singleJob in jobs:
-                    ##Get single job info and append to merge job if requirements fullfilled
-                    alignment, datasetName, singleIOV = singleJob["name"].split("_")[2:]    
-
-                    if int(singleIOV) == IOV and datasetName in config["validations"]["Zmumu"][zmumuType][mergeName]["singles"]:
-                        local["alignments"][alignment]["file"] = singleJob["config"]["output"]
-                        job["dependencies"].append(singleJob["name"])
-
-                mergeJobs.append(job)
+                    local["validation"]["IOV"] = IOV
+                    if "customrighttitle" in local["validation"].keys():
+                        if "IOV" in local["validation"]["customrighttitle"]:
+                            local["validation"]["customrighttitle"] = local["validation"]["customrighttitle"].replace("IOV",str(IOV))
+                    local["output"] = "{}/{}/Zmumu/{}/{}/{}".format(config["LFS"], config["name"], zmumuType, mergeName, IOV) 
+
+                    ##Add global plotting options
+                    if "style" in config.keys():
+                        if "Zmumu" in config['style'].keys():
+                            if zmumuType in config['style']['Zmumu'].keys():
+                                local["style"] = copy.deepcopy(config["style"]["Zmumu"][zmumuType])
+                                if "Rlabel" in local["style"] and "customrighttitle" in local["validation"].keys():
+                                    print("WARNING: custom right label is overwritten by global settings")
+
+                    ##Loop over all single jobs
+                    for singleJob in jobs:
+                        ##Get single job info and append to merge job if requirements fullfilled
+                        _alignment, _singleName, _singleIOV = singleJob["name"].split("_")[2:]
+                        if _singleName in config["validations"]["Zmumu"][zmumuType][mergeName]["singles"]:
+                            if int(_singleIOV) == IOV or (int(_singleIOV) == 1 and _singleName in singlesMC): #matching DATA job or any MC single job 
+                                local["alignments"][_alignment]["file"] = singleJob["config"]["output"]
+                                job["dependencies"].append(singleJob["name"])
+
+                    mergeJobs.append(job)  
 
         jobs.extend(mergeJobs)
 
+    if "trends" in config["validations"]["Zmumu"]:
+        print("[WARNING] Zmumu trends are not implemented yet. Nothing to do here...")
+
     return jobs

Alignment/OfflineValidation/src/PreparePVTrends.cc

@@ -558,10 +558,12 @@ void PreparePVTrends::outputGraphs(const pv::wrappedTrends &allInputs,
   g_asym->SetTitle(label);
 
   // scatter or RMS TH1
-  h_RMS[index] = new TH1F(Form("h_RMS_dz_eta_%s", label.Data()), label, ticks.size() - 1, &(ticks[0]));
+  std::vector<double> newTicks = ticks;
+  newTicks.insert(newTicks.end(), ticks.back() + 1.);
+  h_RMS[index] = new TH1F(Form("h_RMS_dz_eta_%s", label.Data()), label, newTicks.size() - 1, &(newTicks[0]));
   h_RMS[index]->SetStats(kFALSE);
 
-  for (size_t bincounter = 1; bincounter < ticks.size(); bincounter++) {
+  for (size_t bincounter = 1; bincounter < ticks.size() + 1; bincounter++) {
     h_RMS[index]->SetBinContent(
         bincounter, std::abs(allInputs.getHigh()[label][bincounter - 1] - allInputs.getLow()[label][bincounter - 1]));
     h_RMS[index]->SetBinError(bincounter, 0.01);