Merge pull request #42641 from amecca/fixFindAndChange_backport_13_2_X

Fix to findAndChange for the new All-in-one [13.2.x]

Alignment/OfflineValidation/python/TkAlAllInOneTool/findAndChange.py

@@ -1,5 +1,6 @@
 import os
 from FWCore.ParameterSet.pfnInPath import pfnInPath
+import ROOT
 
 ##############################################
 def digest_path(path):
@@ -16,41 +17,38 @@ def digest_path(path):
     if not isinstance(path, str):
         return path
 
+    path_expanded = os.path.expandvars(path)
+
     # split path in folders
     protocol = ""
-    if "://" in path:
-        protocol = path.split("://")[0]+"://"
-        path_s = path.split("://")[1].split(os.sep)
+    if "://" in path_expanded:
+        protocol = path_expanded.split("://")[0]+"://"
+        path_d = path_expanded.split("://")[1]
+    elif ":" in path_expanded:
+        protocol = path_expanded.split(":")[0]+':'
+        path_d = ":".join(path_expanded.split(":")[1:])
+        # Similar to just `split(':')[1]`, but handles the case in which the rest of the path contains one or more ':'
     else:
-        path_s = path.split(os.sep)    
+        path_d = path_expanded
+
+    path_s = path_d.split(os.sep)
 
-    path_d_s = []
     placeholderIdx = []
     for ipart,part in enumerate(path_s):
-        # Look for environment variables such as $CMSSW_BASE
-        if part.startswith('$'):
-            env_var = part[1:].replace('{', '').replace('}', '')
-            path_d_s.append(os.environ[env_var])
         # Look for {} placeholder to be replaced internally 
-        elif "{}" in part:
+        if "{}" in part:
             placeholderIdx.append(ipart)
-            path_d_s.append(part)
-        else:
-            path_d_s.append(part)
 
     # re-join folders into full path 
     # only check path up to first placeholder occurence
-    path_d = os.path.join(*path_d_s)
     if len(placeholderIdx) > 0:
-        path_to_check = os.path.join(*path_d_s[:placeholderIdx[0]])
+        path_to_check = os.path.join(*path_s[:placeholderIdx[0]])
+        # re add front / if needed
+        if path_d.startswith(os.sep):
+            path_to_check = os.sep + path_to_check
     else:
         path_to_check = path_d
 
-    # re add front / if needed
-    if path.startswith(os.sep):
-        path_d = os.sep + path_d
-        path_to_check = os.sep + path_to_check
-
     # check for path to exist
     if not os.path.exists(path_to_check) and "." in os.path.splitext(path_to_check)[-1]:
         # in case of directory pointing to file try backup
@@ -64,14 +62,37 @@ def digest_path(path):
     # if all is OK return path to directory or file
     return path_d
 
+#########################################
+def get_root_color(value):
+#########################################
+    """
+       Returns an integer correspondig to the ROOT color
+    """
+    if(isinstance(value, str)):
+        if(value.isdigit()):
+            return int(value)
+        elif('-' in value):
+            pre, op, post = value.partition('-')
+            return get_root_color(pre.strip()) - get_root_color(post.strip())
+        elif('+' in value):
+            pre, op, post = value.partition('+')
+            return get_root_color(pre.strip()) + get_root_color(post.strip())
+        else:
+            return getattr(ROOT.EColor, value)
+    else:
+        return int(value)
+
 #########################################
 def get_all_keys(var):
 #########################################
     """
        Generate all keys for nested dictionary
+       - reserved keywords are not picked up
     """
+    reserved_keys = ["customrighttitle","title"]
     if hasattr(var,'items'):
         for k, v in var.items():
+            if k in reserved_keys: continue
             if isinstance(v, dict):
                 for result in get_all_keys(v):
                     yield result
@@ -97,10 +118,16 @@ def find_and_change(keys, var, alt=digest_path):
         for key in keys:
             for k, v in var.items():
                 if k == key:
-                    if isinstance(v,list):
-                        var[k] = [alt(_v) for _v in v]
-                    else:
-                        var[k] = alt(v)
+                    if "color" in key:
+                        if isinstance(v,list):
+                            var[k] = [get_root_color(_v) for _v in v]
+                        else:
+                            var[k] = get_root_color(v)
+                    else:    
+                        if isinstance(v,list):
+                            var[k] = [alt(_v) for _v in v]
+                        else:
+                            var[k] = alt(v)
                     yield alt(v)
                 if isinstance(v, dict):
                     for result in find_and_change([key], v):

Alignment/OfflineValidation/scripts/validateAlignments.py

@@ -214,6 +214,7 @@ def main():
             raise Exception("Unknown config extension '{}'. Please use json/yaml format!".format(args.config.split(".")[-1])) 
 
     ##Check for all paths in configuration and attempt to "digest" them
+    ##As a bonus, all ROOT colors are turned to the integer value
     for path in fnc.find_and_change(list(), config):
         if args.verbose and ("." in str(path) or "/" in str(path)):
             print("Digesting path: "+str(path))

Alignment/OfflineValidation/test/examples/DMR_testFeatures.yaml

@@ -0,0 +1,48 @@
+LFS: /eos/cms/store/group/alca_trackeralign/AlignmentValidation/AlignmentValidation/NewTool
+name: DMR_testFeatures
+
+alignments:
+    mp3707withSD:
+        color: kRed + 3  # ROOT color constants are translated into integers; additions and subtractions (the symbols '+' and '-') are evaulated
+        style: 2101
+        title: mp3707+SD
+        globaltag: 130X_dataRun3_HLT_v2
+        conditions:
+            TrackerAlignmentRcd:
+                connect: sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp3707/jobData/jobm/alignments_MP.db
+                tag: Alignments
+            TrackerSurfaceDeformationRcd:
+                connect: sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp3643/jobData/jobm4/alignments_MP.db
+                tag: Deformations
+
+validations:
+    DMR:
+        merge:
+            mergeJob:
+                methods:
+                - median
+                - rmsNorm
+                curves:
+                - plain
+                customrighttitle: "pp collisions (2023) 13.6 TeV"
+                legendoptions:
+                - mean
+                - rms
+                singles:
+                - singleJob
+                minimum: 10
+
+        single:
+            singleJob:
+                IOV:
+                - 370667
+                alignments:
+                - mp3707withSD
+                trackcollection: hltMergedTracks
+                vertexcollection: offlinePrimaryVertices
+                tthrbuilder: WithTrackAngle
+                usePixelQualityFlag: false
+                maxevents: 2000
+                maxtracks: 18000
+                dataset: /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/datasetfiles/Collisions2023/310723/HLTMonitor_Run2023D-Express-v2_FEVTHLTALL/Dataset_Validation_HLTMonitor_Run2023D-Express-v2_FEVTHLTALL_since370667.txt
+                goodlumi: /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/datasetfiles/Collisions2023/310723/Collisions23_13p6TeV_366403_370790_DCSOnly_TkPx.json