Merge branch 'master' into update_disptrkjet_cuts_master

Alignment/HIPAlignmentAlgorithm/python/common_cff_py.txt

@@ -176,7 +176,7 @@ for sra in SelectorRigidAlignables:
    if not tmpstrsrafound:
       tmprigidalignables.append(sra)
    else:
-      print "{} is already in the non-rigid alignables list. Omitting it in the rigid structures to align.".format(tmpstrsra)
+      print("{} is already in the non-rigid alignables list. Omitting it in the rigid structures to align.".format(tmpstrsra))
 SelectorRigidAlignables = tmprigidalignables
 
 process.AlignmentProducer.ParameterBuilder.SelectorBowed = cms.PSet(

Alignment/HIPAlignmentAlgorithm/scripts/makeHippyCampaign.py

@@ -11,22 +11,28 @@
 import subprocess
 import sys
 
-basedir = "/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy"
-
 thisfile = os.path.abspath(__file__)
 
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument("foldername", help="folder name for the campaign.  Example: CRUZET20xy")
   parser.add_argument("--cmssw", default=os.environ["CMSSW_VERSION"])
   parser.add_argument("--scram-arch", default=os.environ["SCRAM_ARCH"])
-  parser.add_argument("--subfolder", default="", help="subfolder within "+basedir+" to make 'foldername' in.")
+  parser.add_argument("--subfolder", default="", help="subfolder within basedir to make 'foldername' in.")
   parser.add_argument("--merge-topic", action="append", help="things to cms-merge-topic within the CMSSW release created", default=[])
   parser.add_argument("--print-sys-path", action="store_true", help=argparse.SUPPRESS) #internal, don't use this
+  parser.add_argument('--basedir', default="/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy")
   args = parser.parse_args()
 
+  basedir = args.basedir
+  if not os.path.exists(basedir):
+    raise FileExistsError("Base Directory does not exist!")
+
+  if basedir[-1] == '/':
+    basedir = basedir[:-1] #No trailing slashed allowed
+
   if args.print_sys_path:
-    print repr(sys.path)
+    print(repr(sys.path))
     return
 
   folder = os.path.join(basedir, args.subfolder, args.foldername)
@@ -100,7 +106,14 @@ def main():
         shutil.copy(os.path.join(HIPAlignmentAlgorithm, "test", "hippysubmittertemplate.sh"), "submit_template.sh")
         os.chmod("submit_template.sh", os.stat("submit_template.sh").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
         subprocess.check_call(["git", "add", "submit_template.sh"])
-
+
+      if not os.path.exists("submit_script.sh"):
+        shutil.copy(os.path.join(HIPAlignmentAlgorithm, "test", "hippysubmitterscript.sh"), "submit_script.sh")
+        os.chmod("submit_script.sh", os.stat("submit_script.sh").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        subprocess.check_call(["git", "add", "submit_script.sh"])
+
+      print("Dumped files into ", folder)
+
       try:
         subprocess.check_output(["git", "diff", "--staged", "--quiet"])
       except subprocess.CalledProcessError:
@@ -128,7 +141,7 @@ def cd(newdir):
 
 def cmsenv():
   output = subprocess.check_output(["scram", "ru", "-sh"])
-  for line in output.split(";\n"):
+  for line in output.decode('utf8').split(";\n"):
     if not line.strip(): continue
     match1 = re.match(r'^export (\w*)="([^"]*)"$', line)
     match2 = re.match(r'^unset *((\w* *)*)$', line)

Alignment/HIPAlignmentAlgorithm/test/hippysubmitterscript.sh

@@ -0,0 +1,15 @@
+#! /bin/bash
+
+hipName="$(grep -m 1 "alignmentname=" $1 | cut -d= -f2)"
+
+if [ -z "$hipName" ]
+then
+    echo "Value for 'alignmentname' not found in template. Please check your submission template."
+else
+    nohup ./$1 >> ../$hipName.log 2>&1 &
+    echo $hipName $! >> ../pid.nohup
+    echo "Please follow the log in '../$hipName.log'. To track progress live, use 'tail -f ../$hipName.log'."
+    echo "The nohup job PID is appended to '../pid.nohup' in case the submission should be killed."
+    echo "You can also use 'ps -ef | grep submit_' to find PIDs of currently running alignments."
+fi
+

Alignment/HIPAlignmentAlgorithm/test/hippysubmittertemplate.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 voms-proxy-info | grep timeleft | grep -v -q 00:00:00 || (echo 'no proxy'; exit 1)
 
-(echo $STY > /dev/null) || (echo "run this on a screen"; exit 1)
+(echo $TMUX > /dev/null) || (echo "run this on a screen"; exit 1)
 
 #hpnumber=
 hptype=hp   #or sm

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h

@@ -116,9 +116,6 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase {
   /// called at end of luminosity block
   void endLuminosityBlock(const edm::EventSetup &) override;
 
-  /*   virtual void beginLuminosityBlock(const edm::EventSetup &setup) {} */
-  /*   virtual void endLuminosityBlock(const edm::EventSetup &setup) {} */
-
   /// Called in order to pass parameters to alignables for a specific run
   /// range in case the algorithm supports run range dependent alignment.
   bool setParametersForRunRange(const RunRange &runrange) override;

Alignment/MillePedeAlignmentAlgorithm/test/test_payload_sanity.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 function die { echo $1: status $2; exit $2; }
 
-INPUTFILE=${SCRAM_TEST_PATH}/alignments_MP.db
-(cmsRun ${SCRAM_TEST_PATH}/AlignmentRcdChecker_cfg.py inputSqliteFile=${INPUTFILE}) || die 'failed running AlignmentRcdChecker'
+echo -e "Content of the current directory is: "`ls .`
+INPUTFILE=alignments_MP.db
+(cmsRun ${SCRAM_TEST_PATH}/AlignmentRcdChecker_cfg.py inputSqliteFile=${INPUTFILE}) || die 'failed running AlignmentRcdChecker' $?
 rm $INPUTFILE

Alignment/MillePedeAlignmentAlgorithm/test/test_pede.sh

@@ -4,7 +4,7 @@ function die { echo $1: status $2; exit $2; }
 LOCAL_TEST_DIR=${SCRAM_TEST_PATH}
 
 clean_up(){
-    echo "cleaning the local test area"
+    echo -e "\nCleaning the local test area"
     rm -fr milleBinary00* 
     rm -fr pedeSteer* 
     rm -fr millepede.*
@@ -13,7 +13,7 @@ clean_up(){
     rm -fr *.dat
     rm -fr *.tar
     rm -fr *.gz
-    rm -fr *.db
+    rm -fr *.dump
 }
 
 if test -f "milleBinary*"; then
@@ -39,10 +39,9 @@ if [ $STATUS -eq 0 ]; then
     echo -e "\n @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
     echo -e " @ MillePede Exit Status: "`cat millepede.end`
     echo -e " @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
-    ## mv the output file to the local test directory for the subsequent payload sanity check
-    mv alignments_MP.db ${LOCAL_TEST_DIR}
     ## clean the house now...
     clean_up
+    echo -e "\nContent of the current directory is: "`ls .`
 else 
   die "SKIPPING test, file ${TESTPACKAGE}.tar not found" 0
 fi

Alignment/OfflineValidation/README.md

@@ -114,5 +114,8 @@ For details read [`README_PV.md`](https://github.com/cms-sw/cmssw/blob/master/Al
 ## JetHT validation
 For details read [`README_JetHT.md`](https://github.com/cms-sw/cmssw/blob/master/Alignment/OfflineValidation/README_JetHT.md)
 
+## MTS validation
+For details read [`README_MTS.md`](https://github.com/cms-sw/cmssw/blob/master/Alignment/OfflineValidation/README_MTS.md)
+
 ## General info about IOV/run arguments
 For details read [`README_IOV.md`](https://github.com/cms-sw/cmssw/blob/master/Alignment/OfflineValidation/README_IOV.md)

Alignment/OfflineValidation/README_MTS.md

@@ -0,0 +1,33 @@
+## MTS (Muon Track Splitting) validation
+
+### General info
+
+```
+validations:
+    MTS:
+        <step_type>:
+            <step_name>:
+                <options>
+```
+
+MTS validation runs in 1 possible type of steps:
+ - single (validation analysis by MTS_cfg.py)
+Step name is arbitrary string which will be used as a reference for consequent steps.
+Merge and trend jobs are not yet implemented.
+
+### Single MTS jobs
+
+Single jobs can be specified per run (IoV as well).
+
+**Parameters below to be updated**
+Variable | Default value | Explanation/Options
+-------- | ------------- | --------------------
+IOV | None | List of IOVs/runs defined by integer value. IOV 1 is reserved for MC.
+Alignments | None | List of alignments. Will create separate directory for each.
+dataset | See defaultInputFiles_cff.py | Path to txt file containing list of datasets to be used. If file is missing at EOS or is corrupted - job will eventually fail (most common issue).
+goodlumi | cms.untracked.VLuminosityBlockRange() | Path to json file containing lumi information about selected IoV - must contain list of runs under particular IoV with lumiblock info. Format: `IOV_Vali_{}.json`
+maxevents | 1 | Maximum number of events before cmsRun terminates.
+trackcollection | "generalTracks" | Track collection to be specified here, e.g. "ALCARECOTkAlMuonIsolated" or "ALCARECOTkAlMinBias" ...
+tthrbuilder | "WithAngleAndTemplate" | Specify TTRH Builder
+usePixelQualityFlag | True | Use pixel quality flag?
+cosmicsZeroTesla | False | Is this validation for cosmics with zero magnetic field?

Alignment/OfflineValidation/bin/BuildFile.xml

@@ -18,5 +18,6 @@
 <bin name="SplitVmerge" file="SplitVmerge.cc,Options.cc" />
 <bin name="Zmumumerge" file="Zmumumerge.cc,Options.cc" />
 <bin name="DiMuonVmerge" file="DiMuonVmerge.cc,Options.cc" />
+<bin name="MTSmerge" file="MTSmerge.cc,Options.cc" />
 <bin name="haddws" file="haddws.C" />
 <bin name="jetHtPlotter" file="jetHtPlotter.cc,JetHtPlotConfiguration.cc,Options.cc" />

Alignment/OfflineValidation/bin/MTSmerge.cc

@@ -0,0 +1,87 @@
+#include <cstdlib>
+#include <string>
+#include <iostream>
+#include <numeric>
+#include <functional>
+
+#include "exceptions.h"
+#include "toolbox.h"
+#include "Options.h"
+
+#include "boost/filesystem.hpp"
+#include "boost/property_tree/ptree.hpp"
+#include "boost/property_tree/json_parser.hpp"
+#include "boost/optional.hpp"
+
+#include "TString.h"
+#include "TASImage.h"
+
+#include "Alignment/OfflineValidation/macros/trackSplitPlot.h"
+#include "Alignment/OfflineValidation/macros/trackSplitPlot.C"
+#include "Alignment/OfflineValidation/interface/TkAlStyle.h"
+
+using namespace std;
+using namespace AllInOneConfig;
+
+namespace pt = boost::property_tree;
+
+int merge(int argc, char* argv[]) {
+  // parse the command line
+
+  Options options;
+  options.helper(argc, argv);
+  options.parser(argc, argv);
+
+  //Read in AllInOne json config
+  pt::ptree main_tree;
+  pt::read_json(options.config, main_tree);
+
+  pt::ptree alignments = main_tree.get_child("alignments");
+  pt::ptree validation = main_tree.get_child("validation");
+  pt::ptree global_style;
+  pt::ptree merge_style;
+
+  int iov = validation.count("IOV") ? validation.get<int>("IOV") : 1;
+  std::string rlabel = validation.count("customrighttitle") ? validation.get<std::string>("customrighttitle") : "";
+  rlabel = merge_style.count("Rlabel") ? merge_style.get<std::string>("Rlabel") : rlabel;
+  std::string cmslabel = merge_style.count("CMSlabel") ? merge_style.get<std::string>("CMSlabel") : "INTERNAL";
+  if (TkAlStyle::toStatus(cmslabel) == CUSTOM)
+    TkAlStyle::set(CUSTOM, NONE, cmslabel, rlabel);
+  else
+    TkAlStyle::set(TkAlStyle::toStatus(cmslabel), NONE, "", rlabel);
+
+  TString filesAndLabels;
+  for (const auto& childTree : alignments) {
+    // Print node name and its attributes
+    std::cout << "Node: " << childTree.first << std::endl;
+    for (const auto& attr : childTree.second) {
+      std::cout << "  Attribute: " << attr.first << " = " << attr.second.data() << std::endl;
+    }
+
+    //std::cout << childTree.second.get<string>("file") << std::endl;
+    //std::cout << childTree.second.get<string>("title") << std::endl;
+    //std::cout << childTree.second.get<int>("color") << std::endl;
+    //std::cout << childTree.second.get<int>("style") << std::endl;
+
+    std::string toAdd = childTree.second.get<string>("file") +
+                        Form("/MTSValidation_%s_%d.root=", childTree.first.c_str(), iov) +
+                        childTree.second.get<string>("title") +
+                        Form("|%i|%i,", childTree.second.get<int>("color"), childTree.second.get<int>("style"));
+    filesAndLabels += toAdd;
+  }
+
+  std::cout << "filesAndLabels: " << filesAndLabels << std::endl;
+
+  TkAlStyle::legendheader = "";
+  TkAlStyle::legendoptions = "all";
+  outliercut = -1.0;
+  //fillmatrix();
+  subdetector = "PIXEL";
+  makePlots(filesAndLabels, "./");
+
+  return EXIT_SUCCESS;
+}
+
+#ifndef DOXYGEN_SHOULD_SKIP_THIS
+int main(int argc, char* argv[]) { return exceptions<merge>(argc, argv); }
+#endif

Alignment/OfflineValidation/interface/TkAlStyle.h

@@ -42,6 +42,9 @@ enum PublicationStatus {
 // Data era: determines labels of data-taking periods, e.g. CRUZET
 enum Era { NONE, CRUZET15, CRAFT15, COLL0T15 };
 
+// Alignment object
+enum AlignObj { IDEALAlign, RUN1Align, CRUZETAlign, CRAFTAlign, Coll0TAlign };
+
 class TkAlStyle {
 public:
   // Adjusts the gStyle settings and store the PublicationStatus
@@ -52,6 +55,13 @@ class TkAlStyle {
   static void set(const TString customTitle);
   static PublicationStatus status() { return publicationStatus_; }
 
+  static TString toTString(const PublicationStatus status);
+  static TString toTString(const Era era);
+  static TString toTString(const AlignObj obj);
+
+  static int color(const AlignObj obj);
+  static int style(const AlignObj obj);
+
   // Draws a title "<CMS label> 2015" on the current pad
   // dependending on the PublicationStatus
   //  INTERNAL    : no extra label (intended for AN-only plots with data)

Alignment/OfflineValidation/macros/FitPVResiduals.C

@@ -39,7 +39,7 @@
 #include <sstream>
 #include <string>
 #include <vector>
-//#include "Alignment/OfflineValidation/macros/TkAlStyle.cc"
+//#include "Alignment/OfflineValidation/interface/TkAlStyle.h"
 #include "Alignment/OfflineValidation/macros/CMS_lumi.h"
 #define PLOTTING_MACRO  // to remove message logger
 #include "Alignment/OfflineValidation/interface/PVValidationHelpers.h"

Alignment/OfflineValidation/macros/momentumElectronBiasValidation.C

@@ -131,7 +131,7 @@ void momentumElectronBiasValidation(TString variable,
   std::vector<TString> labels;  // list of input labels
   if (!checkArguments(
           variable, path, alignmentWithLabel, outputType, radius, verbose, givenMin, givenMax, mode, files, labels))
-    return;
+    exit(EXIT_FAILURE);
   else {
     std::cout << "-> Number of files: " << files.size() << std::endl;
   }