Fixes for tptp_2016 branch

dumpMuPDFfactors.py

@@ -0,0 +1,69 @@
+import os,sys
+execfile("/uscms_data/d3/jmanagan/EOSSafeUtils.py")
+
+dirlist = [ 
+'BprimeBprime_M-1000_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1100_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1200_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1300_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1400_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1500_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1600_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1700_TuneCP5_13TeV-madgraph-pythia8',
+'BprimeBprime_M-1800_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1000_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1100_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1200_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1300_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1400_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1500_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1600_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1700_TuneCP5_13TeV-madgraph-pythia8',
+'TprimeTprime_M-1800_TuneCP5_13TeV-madgraph-pythia8',
+]
+
+from ROOT import TFile, TH1
+
+for sample in dirlist:
+    print('---------------------'+sample+'--------------------------')
+    runlist = EOSlistdir('/store/user/jmanagan/FWLJMET102X_1lep2018mupdf_072919/'+sample+'/singleLep2017/')
+    if len(runlist) > 1: 
+        print('PROBLEM: more than 1 crab directory, SKIPPING')
+        continue
+
+    rfile = TFile.Open('root://cmseos.fnal.gov//store/user/jmanagan/FWLJMET102X_1lep2018mupdf_072919/'+sample+'/singleLep2017/'+str(runlist[0])+'/0000/'+sample+'_1.root')
+    hist = rfile.Get("mcweightanalyzer/weightHist")
+    integral = hist.GetBinContent(5) + hist.GetBinContent(7)
+    newpdf = hist.GetBinContent(2)
+
+    print(str(round(newpdf,3))+'. # from integral '+str(integral))
+
+    muhist = rfile.Get("mcweightanalyzer/muRFcounts")
+    #print('MuRF nominal yield = '+str(round(muhist.GetBinContent(1),3)))
+
+    muvars = []
+    for ibin in range(1,muhist.GetNbinsX()+1): muvars.append(muhist.GetBinContent(ibin))
+    muvars.sort()
+
+    print('MuRF up yield = '+str(round(muvars[6],3)))
+    print('MuRF dn yield = '+str(round(muvars[0],3)))
+
+    pdfhist = rfile.Get("mcweightanalyzer/pdfcounts")
+    #print('PDF nominal yield = '+str(round(pdfhist.GetBinContent(1),3)))
+
+    pdfvars = []
+    for ibin in range(1,pdfhist.GetNbinsX()+1): pdfvars.append(pdfhist.GetBinContent(ibin))
+    pdfvars.sort()
+
+    print('PDF up yield = '+str(round(pdfvars[83],3)))
+    print('PDF dn yield = '+str(round(pdfvars[15],3)))
+
+    print('MUup scale factor = '+str(round(muhist.GetBinContent(1)/muvars[6],3)))
+    print('MUdn scale factor = '+str(round(muhist.GetBinContent(1)/muvars[0],3)))
+    print('PDFup scale factor = '+str(round(pdfhist.GetBinContent(1)/pdfvars[83],3)))
+    print('PDFdn scale factor = '+str(round(pdfhist.GetBinContent(1)/pdfvars[15],3)))
+
+
+
+
+

dumpcounts.py

@@ -0,0 +1,111 @@
+import os,sys
+
+filelist = {
+'BprimeBprime_M-1000_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1100_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':7,
+'BprimeBprime_M-1200_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1300_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1400_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1500_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1600_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':3,
+'BprimeBprime_M-1700_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-1800_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+#'BprimeBprime_M-700_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':3,
+#'BprimeBprime_M-800_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'BprimeBprime_M-900_TuneCUETP8M1_13TeV-madgraph-pythia8_BHBH_1_hadd.root':4,
+'DYJetsToLL_M-50_HT-1200to2500_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'DYJetsToLL_M-50_HT-2500toInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+#'DYJetsToLL_M-50_HT-200to400_TuneCP5_PSweights_13TeV-madgraphMLM-pythia8_1_hadd.root':5,
+'DYJetsToLL_M-50_HT-400to600_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+'DYJetsToLL_M-50_HT-600to800_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'DYJetsToLL_M-50_HT-800to1200_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'QCD_HT1000to1500_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+'QCD_HT1500to2000_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'QCD_HT2000toInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+#'QCD_HT200to300_TuneCP5_13TeV-madgraphMLM-pythia8_1_hadd.root':5,
+'QCD_HT300to500_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':5,
+'QCD_HT500to700_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+'QCD_HT700to1000_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':3,
+'ST_s-channel_4f_leptonDecays_13TeV-amcatnlo-pythia8_hadd.root':0,
+'ST_t-channel_antitop_4f_inclusiveDecays_13TeV_PSweights-powhegV2-madspin_1_hadd.root':4,
+'ST_t-channel_top_4f_inclusiveDecays_13TeV_PSweights-powhegV2-madspin_1_hadd.root':7,
+'ST_tW_antitop_5f_inclusiveDecays_13TeV-powheg-pythia8_TuneCUETP8M2T4_hadd.root':0,
+'ST_tW_top_5f_inclusiveDecays_13TeV-powheg-pythia8_TuneCUETP8M2T4_hadd.root':0,
+
+###### Don't need for 2016 ####
+'TTTo2L2Nu_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt0to700_1_hadd.root':6,
+'TTTo2L2Nu_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt700to1000_1_hadd.root':6,
+'TTTo2L2Nu_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt1000toInf_1_hadd.root':6,
+'TTToHadronic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt0to700_1_hadd.root':6,
+'TTToHadronic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt700to1000_1_hadd.root':6,
+'TTToHadronic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt1000toInf_1_hadd.root':6,
+'TTToSemiLeptonic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt0to700_1_hadd.root':12,
+'TTToSemiLeptonic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt700to1000_1_hadd.root':12,
+'TTToSemiLeptonic_TuneCP5_PSweights_13TeV-powheg-pythia8_Mtt1000toInf_1_hadd.root':12,
+######
+
+'TTWJetsToLNu_TuneCUETP8M1_13TeV-amcatnloFXFX-madspin-pythia8_hadd.root':0,
+'TTZToLLNuNu_M-10_TuneCUETP8M1_13TeV-amcatnlo-pythia8_1_hadd.root':2,
+#'TT_TuneCUETP8M2T4_PSweights_13TeV-powheg-pythia8_Mtt0to700_1_hadd.root':2,
+#'TT_TuneCUETP8M2T4_13TeV-powheg-pythia8_Mtt0to700_hadd.root':0,
+'TprimeTprime_M-1000_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1100_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1200_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1300_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1400_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':3,
+'TprimeTprime_M-1500_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1600_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1700_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-1800_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+#'TprimeTprime_M-700_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+#'TprimeTprime_M-800_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'TprimeTprime_M-900_TuneCUETP8M1_13TeV-madgraph-pythia8_BWBW_1_hadd.root':4,
+'WJetsToLNu_HT-1200To2500_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+'WJetsToLNu_HT-2500ToInf_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'WJetsToLNu_HT-400To600_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_hadd.root':0,
+'WJetsToLNu_HT-600To800_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+'WJetsToLNu_HT-800To1200_TuneCUETP8M1_13TeV-madgraphMLM-pythia8_1_hadd.root':2,
+#'WJetsToLNu_HT-200To400_TuneCP5_13TeV-madgraphMLM-pythia8_1_hadd.root':8,
+'WW_TuneCUETP8M1_13TeV-pythia8_hadd.root':0,
+'WZ_TuneCUETP8M1_13TeV-pythia8_hadd.root':0,
+'ZZ_TuneCUETP8M1_13TeV-pythia8_hadd.root':0,
+'ttHToNonbb_M125_TuneCUETP8M2_ttHtranche3_13TeV-powheg-pythia8_hadd.root':0,
+'ttHTobb_M125_13TeV_powheg_pythia8_hadd.root':0,
+'TT_Mtt-700to1000_TuneCUETP8M2T4_13TeV-powheg-pythia8_hadd.root':0,
+'TT_Mtt-1000toInf_TuneCUETP8M2T4_13TeV-powheg-pythia8_hadd.root':0,
+
+}
+
+from ROOT import TFile, TH1
+eosdir = 'root://cmseos.fnal.gov//store/user/cholz/FWLJMET102X_1lep2016Dnn_06092020_step1hadds/'
+for filekey in sorted(filelist.keys()):
+    print('-------------------------------------------------------')
+    rfile = TFile.Open(eosdir+filekey)
+    hist = rfile.Get("weightHist")
+
+    if filelist[filekey] > 1:
+        print 'Opening ',filelist[filekey],'files:'
+        for ifile in range(2,filelist[filekey]+1):
+            #print 'file #',ifile
+            tempfile = TFile.Open(eosdir+filekey.replace('_1_','_'+str(ifile)+'_'))
+            temphist = tempfile.Get("weightHist")
+	    #print "temphist: ",temphist
+            temphist.SetDirectory(0)
+            hist.Add(temphist)
+            tempfile.Close()                                  
+
+    adjusted = hist.GetBinContent(1)
+    integral = hist.GetBinContent(5) + hist.GetBinContent(7)
+    newpdf = hist.GetBinContent(2)
+
+    if 'prime' not in filekey:
+        print(str(adjusted)+'. # from integral '+str(integral)+', file '+filekey)
+    else:
+        print(str(newpdf)+'. # from integral '+str(integral)+', file '+filekey)
+
+print 'done'
+
+
+
+
+

makeTemplates/README

@@ -0,0 +1,73 @@
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+The main code to make histograms is "doThetaTemplates.py":
+To be modified by user (listed in the order as they appear in the script):
+	- input directory (step1 or step2)
+	- list of samples to be run for signals, backgrounds, and data
+	- group the background processes into "ewk", "qcd", "top", "ttjets", 
+	etc (no need to modify unless samples and thus their names change). Also, note that
+	yields/plotting are setup for "ewk/top/qcd" grouped backgrounds. If detailed background
+	processes wanted, the implementations has to be completed!
+	- systematics: they have to be defined in "analyze.py" and corresponding samples must exist.
+	The systematics can be turned off if only simple plots wanted without total uncertainty bands
+	or only considering the normalization uncertainties in the limit calculation!
+	- Q^2 needs to be taken care of separately; currently (Dec 16, 2015) there is a hack since
+	scaleDown samples does not exist. FIX THIS IF YOU HAVE THE SAMPLE
+	- Give the list of cuts to be applied! New cuts can be included in "cutList" as long as
+	they are also added in "analyze.py" to "cut" string and also add it to "cutString" variable in "makeTemplates.py"
+	- modify the output directory names, etc
+	- modify the normalization uncertainties
+	- give the discriminants in "plotList" (given near the end). In the definition, give a name to distribution 
+	as a key to the dictionary, name of the variable in ljmet trees, x-axis label, and the desired binning
+	- NOTE THAT RENORM/FACT and PDF UNCERTAINTIES ARE NOT NORMALIZED TO NOMINAL!!!
+	USE "splitChannels.py" TO DO THIS. SEE BELOW
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+"analyze.py":
+This does not need a modification, if you are only changing the cuts that are already there and 
+using only the systematics defined there (or not using the systematics). If you want a different
+categorization, this also has to be implemented here!
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+"samples.py":
+It needs definitions of all the samples that are being analyzed. This is for easy access to input files!
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+"weights.py":
+This is where primary MC weights are defined. Each sample again must have a defined weight here with
+number of generated events, x-sec, and the luminosity
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+"doThetaTemplates.py" can be run interactively by simply doing "python doThetaTemplates.py". In
+this case, templates for a single set of cuts will be produced.
+If you want to do a basic cut optimization, use "doCondorThetaTemplates.py" and "doCondorThetaTemplates.sh". 
+Before running condor scripts, "doThetaTemplates.py" has to be modified with the configuration wanted.
+If additional cuts wanted to be optimized, this can be easily implemented by following the same 
+implementation procedure as the others!
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+Once the templates are produced, there are number of things you can do before limit calculation.
+1) Plotting the theta templates using "plotThetaTemplates.py":
+	- give luminosity info, discriminant name and the cut string that you want to plot
+	- give 2 signals to show on the stack plots
+	- do you have the systematics in the templates? If so, which ones (give the same names as the ones used in 
+	"analyze.py")? Turn off the bools if you don't have the shapes
+	- if you have the shape systematics, you can produce two types of plots; one with the uncertainty bands are separated in
+	the lower panel and another one with only the total uncertainty band using "doOneBand".
+	- template directory needs to be specified.
+	- check if you have the right normalization systematics, they will be added to error bands
+	- note that the "QCD" process will not be shown on the plots if it is less than 0.5% of the background
+	- if you'd like to change plot labels, etc, use "formatLowerHist" and "formatUpperHist"
+	- "isRebinned" is a postfix that is attached to root file names when you rebin the histograms or do other things!
+
+2) Rebinning the theta templates using "modifyBinning.py":
+	- give the template directory information.
+	- this script can modify the binning for a given statistical uncertainty threshold, 
+	but the implementation hasn't been completely tested, so use at your own risk.
+	- You can also rebin the histograms by giving a custom binning choice in "xbinsList"
+
+3) "splitChannels.py":
+	- This script can be used for multiple purposes: For example,
+		* split templates into different channels (to run limits separately)
+		* remove some of the shape uncertainties (for debugging the results)
+		* normalize some of the shape uncertainties in order to use only the shape (for example
+		muF, muR, and pdf are not normalized to nominal in the template maker, so it has to be done here!!)
+		* there are examples in the script showing how to do these
+	- give the template directory information.
+	- use the rebinned templates for this
+	- give a unique postfix to files for each configuration instead of just "modified"
+