fix: Merge with new JEC/jet map features

feat: new xsec, add assertion
- helper:add new xs
- scripts:assertion
- scripts: hadd

feat: merge array_writer

- utils: add array_writer
- wf: merge Wc/tt and rename, merge e/mu DY, ttdilep
- wf: use array writer

quickcheck.py

@@ -0,0 +1,28 @@
+import matplotlib.pyplot as plt
+import mplhep as hep
+from coffea.util import load
+
+out = load("hists_ctag_Wc_sf_test/hists_ctag_Wc_sf_test.coffea")
+for i in range(4):
+    hep.histplot(
+        out[list(out.keys())[0]]["weird_MET_phi"][i, :],
+        label=out[list(out.keys())[0]]["weird_MET_phi"].axes[0].value(i),
+    )
+plt.legend()
+plt.savefig("all_MET_phi.pdf")
+plt.cla()
+for i in range(4):
+    hep.histplot(
+        out[list(out.keys())[0]]["weird_MET_pt"][i, :],
+        label=out[list(out.keys())[0]]["weird_MET_pt"].axes[0].value(i),
+    )
+plt.legend()
+plt.savefig("all_MET_pt.pdf")
+plt.cla()
+
+print(out[list(out.keys())[0]]["MET_phi"])
+hep.histplot(out[list(out.keys())[0]]["MET_phi"][0, sum, :])
+plt.savefig("selcted_phi.pdf")
+plt.cla()
+hep.histplot(out[list(out.keys())[0]]["MET_pt"][0, sum, :])
+plt.savefig("selcted_pt.pdf")

runner.py

@@ -86,15 +86,15 @@ def get_main_parser():
             "2017_UL",
             "2016preVFP_UL",
             "2016postVFP_UL",
-            "prompt_dataMC",
+            "CAMPAIGN_prompt_dataMC",
         ],
         help="Dataset campaign, change the corresponding correction files",
     )
     parser.add_argument(
         "--isSyst",
         default=False,
         type=str,
-        choices=[False, "all", "weight_only", "JERC_split"],
+        choices=[False, "all", "weight_only", "JERC_split", "JP_MC"],
         help="Run with systematics, all, weights_only(no JERC uncertainties included),JERC_split, None",
     )
     parser.add_argument("--isArray", action="store_true", help="Output root files")
@@ -235,8 +235,8 @@ def get_main_parser():
     if args.output == parser.get_default("output"):
         index = args.samplejson.rfind("/") + 1
         sample_json = args.samplejson[index:]
-        histoutdir = f"hists_{args.workflow}"
-        outdir = f"arrays_{args.workflow}"
+        histoutdir = f"hists_{args.workflow}_{sample_json.rstrip('.json')}"
+        outdir = f"arrays_{args.workflow}_{sample_json.rstrip('.json')}"
         coffeaoutput = (
             f'{histoutdir}/hists_{args.workflow}_{(sample_json).rstrip(".json")}.coffea'
         )
@@ -262,12 +262,12 @@ def get_main_parser():
     if args.only is not None:
         if args.only in sample_dict.keys():  # is dataset
             sample_dict = dict([(args.only, sample_dict[args.only])])
-            coffeaoutput = coffeaoutput.replace(".coffea", f"_{key}.coffea")
+            coffeaoutput = coffeaoutput.replace(".coffea", f"_{args.only}.coffea")
         elif args.only.isdigit():
             isamp = int(args.only)
             nsamp = len(sample_dict.keys())
             if isamp >= nsamp:
-                raise RuntimeError(
+                print(
                     f"There are {nsamp} datasets, please use --only n with n<{nsamp}."
                 )
             key = list(sample_dict.keys())[isamp]
@@ -350,7 +350,10 @@ def get_main_parser():
         raise Exception(f"{coffeaoutput} exists")
 
     if args.isArray:
-        os.system(f"mkdir -p {outdir}")
+        if path.exists(outdir) and args.overwrite == False and args.only is None:
+            raise Exception("Directory exists")
+        else:
+            os.system(f"mkdir -p {outdir}")
 
     if args.executor not in ["futures", "iterative", "dask/lpc", "dask/casa"]:
         """

scripts/dohadd.py

@@ -0,0 +1,25 @@
+import os, sys
+from glob import glob
+
+indir = sys.argv[1]
+
+systs = os.listdir(indir)
+
+outfile = open("hadd.sh", "w")
+
+for syst in systs:
+    roots = glob(f"{indir}/{syst}/*/*.root")
+    if len(roots) == 0:
+        print(f"Skipping {indir}/{syst}. Not the right directory structure.")
+        continue
+    samps = os.listdir(f"{indir}/{syst}")
+    for samp in samps:
+        if len(glob(f"{indir}/{syst}/{samp}/*.root")) == 0:
+            continue
+        outfile.write(
+            f"hadd -v 0 {indir}/{syst}/{samp}.root {indir}/{syst}/{samp}/*.root\n"
+        )
+
+print(
+    "Now run `parallel :::: hadd.sh` from an environment with ROOT installed. E.g. \nconda activate rootenv\nparallel :::: hadd.sh\nconda activate btv_coffea"
+)

scripts/dump_prescale.py

@@ -25,9 +25,7 @@ def get_prescale(HLT, lumimask, verbose=False):
     if not os.path.exists(
         f"src/BTVNanoCommissioning/data/Prescales/HLTinfo_{HLT}_run{runs[0]}_{runs[-1]}.csv"
     ):
-
         for run in runs:
-
             os.system(
                 f"brilcalc trg --prescale --hltpath 'HLT_{HLT}*'  -r {run}  --output-style csv &>tmp.csv"
             )

scripts/fetch.py

@@ -62,6 +62,23 @@
     action="store_true",
     default=False,
 )
+parser.add_argument(
+    "-r",
+    "--redirector",
+    help="xrootd ridirector in case sites are not found",
+    choices=["infn", "fnal", "cern"],
+    default="infn",
+)
+parser.add_argument(
+    "-j", "--ncpus", help="Number of CPUs to use for validation", default="4"
+)
+parser.add_argument(
+    "--skipvalidation",
+    action="store_true",
+    help="If true, the readability of files will not be validated.",
+    default=False,
+)
+
 parser.add_argument("--campaign", help="campaign info", default=None, type=str)
 
 
@@ -210,9 +227,14 @@ def getFilesFromDas(args):
 
         if xrd is None:
             print(
-                f"No SITE available in the whitelist for file {dsname}, change to global redirector"
+                f"No SITE available in the whitelist for file {dsname}, change to global redirector: {args.redirector}"
             )
-            xrd = "root://xrootd-cms.infn.it//"
+            redirector = {
+                "infn": "root://xrootd-cms.infn.it//",
+                "fnal": "root://cmsxrootd.fnal.gov/",
+                "cern": "root://cms-xrd-global.cern.ch/",
+            }
+            xrd = redirector[args.redirector]
         if args.limit is not None:
             flist = flist[: args.limit]
         if dsname not in fdict:
@@ -335,7 +357,7 @@ def remove_bad_files(sample_dict, outname, remove_bad=True):
         _rmap = p_map(
             validate,
             sample_dict[sample],
-            num_cpus=4,
+            num_cpus=int(args.ncpus),
             desc=f"Validating {sample[:20]}...",
         )
 
@@ -419,7 +441,8 @@ def main(args):
             empty = False
     assert empty, "you have empty lists"
     output_file = "./%s" % (args.output)
-    # fdict = remove_bad_files(fdict, args.output, True)  # remove bad files
+    if not args.skipvalidation:
+        fdict = remove_bad_files(fdict, args.output, True)  # remove bad files
     with open(output_file, "w") as fp:
         json.dump(fdict, fp, indent=4)
         print("The file is saved at: ", output_file)

scripts/plotdataMC.py

@@ -1,5 +1,5 @@
 import numpy as np
-import argparse, os, arrow, glob, re
+import argparse, os, arrow, glob, re, sys
 from coffea.util import load
 import matplotlib.pyplot as plt
 from matplotlib.offsetbox import AnchoredText
@@ -195,8 +195,12 @@
 else:
     var_set = args.variable.split(",")
 for index, discr in enumerate(var_set):
-    if not isinstance(collated["mc"][discr], hist.hist.Hist):
-        continue
+    try:
+        if not isinstance(collated["mc"][discr], hist.hist.Hist):
+            continue
+    except:
+        print(f"{discr} not found. Variable must be in", collated["mc"].keys())
+        sys.exit(1)
     ## remove empty
     if (
         discr not in collated["mc"].keys()

setup.cfg

@@ -54,7 +54,7 @@ where = src
 [options.extras_require]
 dev =
     pytest>=6
-    black>=23.1.0
+    black==24.2.0
 # docs =
 #     Sphinx~=3.0
 #     myst_parser>=0.13

src/BTVNanoCommissioning/data/lumiMasks/Cert_Collisions2024_378981_380649_Golden.json

@@ -0,0 +1,111 @@
+{
+  "378985": [[57, 824]],
+  "378993": [[121, 200]],
+  "379006": [[1, 23]],
+  "379011": [[1, 399]],
+  "379012": [[1, 73]],
+  "379028": [[66, 599]],
+  "379031": [[1, 289]],
+  "379058": [[38, 863]],
+  "379075": [[33, 407]],
+  "379154": [[17, 1065]],
+  "379238": [[11, 296]],
+  "379243": [[39, 337]],
+  "379252": [[34, 677]],
+  "379253": [[1, 104]],
+  "379338": [[1, 170]],
+  "379355": [[1, 20]],
+  "379416": [
+    [1, 34],
+    [56, 88],
+    [94, 102],
+    [107, 225]
+  ],
+  "379420": [[38, 497], [499, 554]],
+  "379425": [[29, 231]],
+  "379433": [[27, 47]],
+  "379442": [[19, 287]],
+  "379454": [[33, 530]],
+  "379456": [[1, 1199]],
+  "379470": [[5, 804]],
+  "379530": [[30, 622]],
+  "379613": [[38, 157]],
+  "379617": [[1, 130], [132, 665]],
+  "379618": [[1, 360]],
+  "379660": [[60, 1566]],
+  "379661": [[1, 406]],
+  "379729": [[57, 668], [670, 1522]],
+  "379765": [[43, 682], [684, 1738]],
+  "379774": [[29, 213]],
+  "379866": [[26, 1375]],
+  "379984": [[36, 948]],
+  "380001": [[36, 616], [618, 727]],
+  "380005": [
+    [33, 79],
+    [82, 311],
+    [317, 1101]
+  ],
+  "380007": [[1, 129]],
+  "380029": [[35, 89]],
+  "380030": [[1, 346]],
+  "380032": [[149, 167]],
+  "380033": [[1, 53]],
+  "380043": [[33, 932]],
+  "380049": [[33, 903]],
+  "380051": [[1, 735]],
+  "380052": [[1, 12]],
+  "380053": [[1, 128]],
+  "380056": [[1, 168]],
+  "380066": [[34, 585]],
+  "380074": [[29, 840], [842, 1335]],
+  "380115": [[27, 1818]],
+  "380126": [[40, 991]],
+  "380127": [[1, 535]],
+  "380128": [[1, 387]],
+  "380195": [[87, 93]],
+  "380197": [[1, 357]],
+  "380235": [[32, 287]],
+  "380236": [[1, 52], [59, 63]],
+  "380306": [[28, 273]],
+  "380308": [[1, 35]],
+  "380309": [[1, 351]],
+  "380310": [[1, 1131]],
+  "380346": [[34, 334], [342, 606]],
+  "380348": [[1, 341]],
+  "380349": [[1, 695]],
+  "380360": [
+    [32, 147],
+    [149, 954],
+    [967, 1972]
+  ],
+  "380377": [[36, 455]],
+  "380384": [[48, 798]],
+  "380385": [[1, 1247]],
+  "380399": [[38, 531]],
+  "380403": [[9, 259]],
+  "380444": [[4, 328]],
+  "380446": [[1, 321], [327, 1033]],
+  "380447": [[1, 810]],
+  "380466": [[37, 400], [416, 555]],
+  "380470": [[1, 1634]],
+  "380481": [[25, 884]],
+  "380513": [[5, 1162]],
+  "380517": [[1, 39]],
+  "380531": [
+    [28, 355],
+    [362, 913],
+    [919, 1062]
+  ],
+  "380534": [[1, 746]],
+  "380537": [[1, 58]],
+  "380538": [[1, 144]],
+  "380564": [[38, 779]],
+  "380567": [[1, 1081]],
+  "380623": [[30, 417]],
+  "380624": [[1, 809]],
+  "380625": [[1, 7]],
+  "380626": [[1, 9], [13, 17]],
+  "380627": [[1, 808]],
+  "380647": [[35, 409]],
+  "380649": [[1, 401]]
+}

src/BTVNanoCommissioning/helpers/definitions.py

@@ -6216,7 +6216,6 @@ def SV_definitions():
 
 
 def axes_name(var):
-
     output = {}
 
     unit = ""