From c11c7639ccf2940ffb3eb6f27b21e63d5009590c Mon Sep 17 00:00:00 2001 From: mteroerd Date: Wed, 31 Jul 2019 16:29:56 +0200 Subject: [PATCH 1/2] Added trend plots, threading in autoSubmitter, fixed some bugs --- .../APEEstimation/data/lumiperrun2017.txt | 963 ++++++++++++++++++ .../APEEstimation/data/lumiperrun2018.txt | 551 ++++++++++ .../test/autoSubmitter/autoSubmitter.py | 339 +++--- .../autoSubmitter/autoSubmitterTemplates.py | 20 +- .../test/autoSubmitter/config.ini | 8 +- .../test/autoSubmitter/helpers.py | 135 +++ .../APEEstimation/test/batch/startSkim.py | 4 +- .../test/cfgTemplate/apeEstimator_cfg.py | 89 +- .../test/plottingTools/drawIterations.py | 8 +- .../test/plottingTools/drawResults.py | 15 +- .../test/plottingTools/drawTrend.py | 37 + .../test/plottingTools/resultPlotter.py | 105 +- .../test/plottingTools/trendPlotter.py | 307 ++++++ 13 files changed, 2271 insertions(+), 310 deletions(-) create mode 100644 Alignment/APEEstimation/data/lumiperrun2017.txt create mode 100644 Alignment/APEEstimation/data/lumiperrun2018.txt create mode 100644 Alignment/APEEstimation/test/autoSubmitter/helpers.py create mode 100644 Alignment/APEEstimation/test/plottingTools/drawTrend.py create mode 100644 Alignment/APEEstimation/test/plottingTools/trendPlotter.py diff --git a/Alignment/APEEstimation/data/lumiperrun2017.txt b/Alignment/APEEstimation/data/lumiperrun2017.txt new file mode 100644 index 0000000000000..18d536a105712 --- /dev/null +++ b/Alignment/APEEstimation/data/lumiperrun2017.txt @@ -0,0 +1,963 @@ +290543 0 +294927 0.008 +294928 0.012 +294929 0.01 +294931 0.011 +294932 0.01 +294933 0.01 +294934 0.009 +294935 0.011 +294936 0.01 +294937 0.009 +294939 0.008 +294940 0.008 +294947 0.008 +294949 0.006 +294950 0.004 +294951 0.004 +294952 0.004 +294953 0.004 +294954 0.004 +294955 0.004 +294956 0.004 +294957 0.003 +294960 0.003 +294986 0.008 +294987 0.006 +294988 0.004 +294990 0.009 +294992 0 +294993 0.007 +294995 0.007 +294997 0.007 +294999 0.006 +295001 0.009 +295122 0.019 +295123 0.017 +295124 0.016 +295125 0.017 +295126 0.015 +295127 0.017 +295128 0.212 +295129 0.004 +295130 0.01 +295131 0.143 +295132 0.003 +295133 0.008 +295134 0.004 +295135 0.028 +295192 0.006 +295193 0.002 +295194 0.018 +295197 0.015 +295198 0.001 +295199 0.009 +295200 0.016 +295201 0.014 +295202 0.013 +295203 0.015 +295204 0.019 +295208 0 +295209 0.157 +295210 0.063 +295315 0.133 +295318 0.208 +295319 0.003 +295320 0 +295321 0 +295322 0.037 +295323 0.082 +295324 0.11 +295325 0.041 +295326 0.032 +295327 0.043 +295328 0.167 +295329 0.023 +295330 0.18 +295331 0.045 +295332 0.131 +295334 0.068 +295335 0.05 +295336 0.004 +295337 0.019 +295339 0.116 +295340 0.619 +295341 0.062 +295342 0.055 +295343 0.039 +295344 0.056 +295345 0.051 +295346 0.016 +295347 0.066 +295348 0.087 +295349 0.005 +295371 0.105 +295376 0.906 +295377 0.091 +295378 0.092 +295379 0.087 +295380 0.093 +295381 1.261 +295390 1.363 +295391 0.676 +295392 0.119 +295393 0.178 +295394 0 +295395 0.773 +295436 1.29 +295437 0.542 +295438 0.519 +295439 0.474 +295440 0.497 +295441 0.001 +295442 0 +295443 0.001 +295444 0.336 +295445 0.64 +295446 0.545 +295447 0.49 +295448 0.47 +295449 0.462 +295450 0.464 +295451 0.159 +295452 0.057 +295453 0.494 +295454 0.484 +295455 0.456 +295456 0.456 +295457 0.448 +295458 0.49 +295459 0.475 +295460 0.316 +295463 25.302 +295600 0.822 +295602 0.411 +295603 2.223 +295604 0.129 +295605 7.942 +295606 2.416 +295607 0.226 +295608 1.897 +295610 1.36 +295613 7.338 +295628 0.787 +295632 0.421 +295634 0.093 +295635 0.098 +295636 0.143 +295637 0.159 +295638 0.321 +295639 0.344 +295640 0.338 +295641 0.336 +295642 0.338 +295644 0.511 +295645 0.328 +295646 0.363 +295647 0.302 +295648 0.309 +295649 0.326 +295650 0.328 +295651 0.105 +295652 0.068 +295655 8.232 +295953 4.216 +295969 0.032 +295976 0.006 +295977 0.318 +296070 1.214 +296071 0.005 +296073 0.029 +296074 0.759 +296076 0.651 +296077 0.723 +296078 0.64 +296079 0.716 +296081 0.537 +296082 0.083 +296083 0.698 +296084 0.683 +296085 0.645 +296086 0.026 +296087 0.598 +296088 0.668 +296089 0.654 +296090 0.477 +296091 0.746 +296092 0.735 +296093 0.64 +296094 0.011 +296095 0.695 +296096 0.643 +296097 0.611 +296098 0.683 +296099 0.538 +296100 0.58 +296101 0.586 +296102 0.028 +296103 0.683 +296104 0.816 +296107 0.54 +296108 0.578 +296109 0.578 +296110 0.556 +296111 0.574 +296112 0.556 +296113 0.567 +296114 1.134 +296115 0.885 +296116 5.767 +296168 1.274 +296172 1.862 +296173 43.167 +296174 1.84 +296641 0.001 +296642 0.018 +296643 0.008 +296644 0.007 +296646 0.008 +296647 0.007 +296663 2.061 +296664 1.863 +296665 1.375 +296666 1.004 +296668 0.939 +296669 1.117 +296671 0 +296674 1.37 +296675 0.955 +296676 1.351 +296677 2.217 +296678 1.12 +296679 0.958 +296680 1.115 +296702 12.152 +296786 77.551 +296787 7.424 +296788 4.713 +296789 1.232 +296790 1.077 +296791 0.307 +296795 1.491 +296796 5.846 +296797 1.279 +296799 0.252 +296800 1.126 +296801 0.438 +296802 1.22 +296866 3.785 +296867 15.121 +296868 1.844 +296869 1.622 +296870 1.61 +296871 1.633 +296872 1.496 +296873 1.487 +296874 1.556 +296875 1.343 +296876 1.484 +296877 1.513 +296878 1.373 +296879 1.584 +296880 4.109 +296881 0.791 +296887 46.81 +296888 20.177 +296895 0.588 +296897 5.519 +296898 2.985 +296899 1.319 +296900 2.672 +296901 1.454 +296902 1.284 +296966 18.559 +296967 3.178 +296968 2.053 +296969 9.912 +296970 23.046 +296971 2.584 +296972 20.646 +296976 18.109 +296977 51.17 +296978 13.078 +296979 1.074 +296980 12.322 +297003 9.149 +297004 2.559 +297006 2.156 +297007 2.153 +297009 1.874 +297010 2.301 +297011 2.155 +297012 2.143 +297015 2.174 +297016 2.101 +297017 1.943 +297018 2.209 +297019 2.314 +297047 4.239 +297048 2.194 +297049 2.231 +297050 103.287 +297056 28.463 +297057 116.596 +297099 7.258 +297100 71.52 +297101 162.004 +297113 50.809 +297114 32.427 +297168 3.591 +297169 2.064 +297170 1.279 +297171 14.297 +297175 15.606 +297176 42.254 +297177 30.027 +297178 200.45 +297179 9.737 +297180 13.682 +297181 6.618 +297211 11.007 +297215 12.017 +297218 3.629 +297219 427.13 +297224 13.007 +297225 2.256 +297227 22.868 +297281 12.215 +297282 4.326 +297283 4.184 +297284 4.292 +297285 3.937 +297286 5.335 +297287 4.005 +297288 3.869 +297289 3.82 +297290 3.722 +297291 4.224 +297292 171.087 +297293 27.926 +297296 85.852 +297308 9.603 +297359 40.936 +297411 202.088 +297424 31.217 +297425 41.21 +297426 26.522 +297429 17.174 +297430 45.888 +297431 39.065 +297432 25.555 +297433 29.723 +297434 28.822 +297435 16.903 +297467 32.177 +297468 22.624 +297469 18.964 +297474 6.301 +297483 48.97 +297484 57.036 +297485 98.144 +297486 123.975 +297487 97.387 +297488 56.849 +297494 5.615 +297495 4.939 +297496 6.84 +297497 4.57 +297498 4.538 +297499 4.547 +297501 4.728 +297502 4.607 +297503 186.983 +297504 10.96 +297505 74.619 +297557 109.55 +297558 67.211 +297562 72.541 +297563 46.549 +297598 1.622 +297599 106.449 +297603 98.936 +297604 82.157 +297605 71.813 +297606 36.83 +297620 82.18 +297656 115.925 +297657 0.868 +297658 0.018 +297659 3.272 +297660 51.159 +297661 21.801 +297662 3.508 +297663 3.634 +297664 7.139 +297665 58.59 +297666 23.377 +297670 8.107 +297671 12.316 +297672 4.484 +297673 4.249 +297674 52.881 +297675 121.338 +297678 0.044 +297722 92.869 +297723 62.076 +298653 0.001 +298678 0.004 +298996 23.753 +298997 5.425 +298998 0.218 +299000 8.544 +299042 2.419 +299061 67.1 +299062 60.61 +299064 16.685 +299065 60.938 +299067 70.948 +299096 25.083 +299149 111.906 +299178 16.895 +299180 17.228 +299183 0.021 +299184 126.94 +299185 24.172 +299316 11.508 +299317 8.984 +299318 11.713 +299324 4.686 +299325 45.922 +299326 5.095 +299327 15.596 +299329 38.663 +299368 33.96 +299369 80.807 +299370 156.517 +299380 56.408 +299381 12.426 +299394 6.195 +299395 40.232 +299396 17.804 +299420 13.825 +299443 36.517 +299450 13.221 +299477 12.124 +299478 46.532 +299479 30.7 +299480 169.715 +299481 200.45 +299592 27.149 +299593 209.696 +299594 62.037 +299595 25.696 +299597 87.687 +299614 61.366 +299616 75.109 +299617 8.778 +299649 91.96 +299996 0.054 +300018 0.018 +300027 0.007 +300043 0.021 +300079 1.713 +300087 13.598 +300088 0.377 +300105 1.265 +300106 20.138 +300107 14.149 +300117 7.067 +300122 302.734 +300123 103.368 +300124 78.307 +300155 288.121 +300156 12.014 +300157 172.245 +300226 119.494 +300233 32.557 +300234 14.933 +300235 50.549 +300236 49.894 +300237 166.194 +300238 56.428 +300239 31.735 +300240 48.886 +300280 158.246 +300281 1.608 +300282 43.362 +300283 5.862 +300284 149.167 +300364 4.47 +300365 3.645 +300366 2.976 +300367 3.103 +300368 3.066 +300369 3.316 +300370 3.263 +300371 3.042 +300372 2.041 +300373 3.194 +300374 3.211 +300375 25.391 +300389 32.939 +300390 3.493 +300391 3.342 +300392 3.448 +300393 3.507 +300394 3.365 +300395 3.412 +300396 3.324 +300397 3.527 +300398 3.624 +300399 4.285 +300400 162.809 +300401 120.559 +300459 83.07 +300461 23.965 +300462 23.548 +300463 30.979 +300464 59.54 +300466 121.62 +300467 80.407 +300497 41.242 +300498 0.018 +300499 0.003 +300500 0.007 +300514 30.005 +300515 263.831 +300516 19.408 +300517 152.714 +300538 10.382 +300539 60.775 +300545 25.888 +300548 3.259 +300551 23.509 +300552 13.394 +300558 118.955 +300560 138.758 +300574 24.28 +300575 12.365 +300576 274.098 +300631 47.336 +300632 4.123 +300633 122.455 +300635 42.953 +300636 275.562 +300673 8.29 +300674 7.954 +300675 2.935 +300676 7.686 +300742 89.558 +300777 132.465 +300780 80.813 +300785 219.585 +300806 55.423 +300811 148.055 +300812 15.036 +300816 36.65 +300817 108.228 +301046 4.405 +301086 19.941 +301141 1.287 +301142 97.836 +301161 99.822 +301165 15.833 +301179 2.375 +301180 15.075 +301183 46.445 +301281 19.248 +301283 122.07 +301298 158.456 +301323 168.528 +301330 66.196 +301359 58.718 +301383 0.47 +301384 95.893 +301391 35.555 +301392 111.762 +301393 2.806 +301394 0.016 +301395 0.023 +301396 3.781 +301397 95.503 +301398 0.565 +301399 13.135 +301417 70.44 +301447 81.522 +301448 48.686 +301449 49.891 +301450 20.446 +301461 99.769 +301472 140.531 +301473 0.006 +301474 0.026 +301475 1.568 +301476 108.771 +301480 18.359 +301519 34.172 +301524 42.824 +301525 8.266 +301528 2.16 +301529 6.66 +301530 14.509 +301531 48.917 +301532 63.676 +301567 64.474 +301627 133.518 +301664 71.724 +301665 72.371 +301694 18.602 +301912 45.227 +301913 6.036 +301914 36.061 +301941 75.901 +301959 279.448 +301960 15.326 +301969 6.769 +301970 23.879 +301984 56.817 +301985 60.906 +301986 55.046 +301987 124.204 +301997 66.23 +301998 221.828 +302019 9.734 +302026 5.245 +302027 0 +302029 15.525 +302030 0 +302031 130.848 +302033 5.23 +302034 0.489 +302036 4.152 +302037 0.908 +302038 0.581 +302040 21.172 +302041 7.814 +302042 55.478 +302043 22.045 +302131 117.987 +302159 16.589 +302163 148.603 +302165 6.928 +302166 2.037 +302225 117.852 +302228 39.543 +302229 68.349 +302239 0.003 +302240 140.805 +302262 77.401 +302263 167.241 +302277 55.33 +302279 5.955 +302280 13.077 +302322 99.769 +302328 97.878 +302337 21.149 +302342 9.609 +302343 15.644 +302344 70.461 +302349 0.005 +302350 17.992 +302388 131.17 +302392 77.275 +302393 160.822 +302448 315.761 +302472 158.522 +302473 61.522 +302474 44.088 +302475 0.101 +302476 34.178 +302479 70.763 +302484 38.163 +302485 176.631 +302486 0.006 +302487 0.008 +302488 0.408 +302489 0.043 +302490 0.004 +302491 0.005 +302492 8.517 +302493 0.144 +302494 87.877 +302509 13.949 +302513 12.88 +302522 0.997 +302523 10.987 +302525 147.326 +302526 89.629 +302548 16.283 +302550 0.797 +302551 0.115 +302553 37.757 +302554 0.093 +302555 2.071 +302563 0.652 +302564 0.005 +302565 0.153 +302566 14.045 +302567 7.455 +302569 0.042 +302570 8.084 +302571 7.438 +302572 55.837 +302573 196.123 +302596 179.171 +302597 148.879 +302634 123.737 +302635 193.516 +302646 0.214 +302651 30.713 +302654 65.319 +302660 0.434 +302661 9.746 +302663 137.094 +303790 0 +303818 0.034 +303819 0.88 +303824 102.249 +303825 24.108 +303832 334.821 +303838 402.09 +303885 415.988 +303948 386.02 +303989 12.366 +303998 76.523 +303999 179.14 +304000 11.033 +304062 386.349 +304119 21.934 +304120 72.611 +304125 342.197 +304144 540.104 +304158 377.191 +304169 341.853 +304170 77.44 +304196 4.138 +304197 2.284 +304198 5.563 +304199 0.618 +304200 88.753 +304204 154.25 +304209 123.533 +304291 3.489 +304292 359.515 +304333 370.332 +304354 68.97 +304366 424.19 +304446 19.422 +304447 371.877 +304448 1.229 +304449 0.265 +304451 7.492 +304452 2.272 +304505 8.024 +304506 119.703 +304507 50.757 +304508 250.911 +304562 223.99 +304616 257.048 +304625 136.778 +304626 2.606 +304654 174.361 +304655 232.686 +304661 55.642 +304662 43.532 +304663 177.925 +304671 309.677 +304672 13.451 +304737 24.6 +304738 410.97 +304739 1.872 +304740 40.992 +304776 9.743 +304777 158.495 +304778 269.721 +304797 565.699 +304911 0 +305040 32.259 +305043 17.652 +305044 99.781 +305045 203.411 +305046 112.571 +305059 158.567 +305062 1.305 +305063 9.589 +305064 383.454 +305081 315.699 +305112 401.878 +305113 12.603 +305114 87.379 +305178 17.131 +305179 3.958 +305180 0.588 +305181 0.582 +305182 1.007 +305183 77.312 +305184 0.102 +305185 0.359 +305186 122.53 +305188 200.223 +305202 208.533 +305204 256.702 +305207 318.551 +305208 83.641 +305234 13.245 +305236 6.486 +305237 338.407 +305247 122.804 +305248 258.32 +305249 9.47 +305250 2.438 +305252 96.572 +305282 46.289 +305310 129.589 +305311 47.358 +305312 68.317 +305313 170.874 +305314 76.471 +305336 66.635 +305338 33.083 +305341 151.794 +305349 3.899 +305350 3.916 +305351 181.03 +305358 50.26 +305364 30.795 +305365 250.672 +305366 236.849 +305376 33.626 +305377 400.511 +305405 160.26 +305406 321.667 +305440 83.376 +305441 36.888 +305516 180.437 +305517 33.976 +305518 235.522 +305586 186.703 +305588 33.842 +305589 163.428 +305590 156.352 +305636 529.294 +305766 266.181 +305809 46.645 +305814 449.3 +305821 240.696 +305832 58.679 +305840 252.297 +305841 0.02 +305842 116.491 +305862 202.42 +305898 230.591 +305902 154.848 +305967 10.514 +306029 9.661 +306030 34.817 +306036 0.372 +306037 12.217 +306038 43.987 +306041 99.914 +306042 100.158 +306048 32.17 +306049 75.553 +306051 73.142 +306091 205.228 +306092 221.438 +306093 0.029 +306095 76.325 +306121 28.879 +306122 40.472 +306125 567.75 +306126 31.65 +306134 7.31 +306135 345.123 +306137 1.093 +306138 212.679 +306139 102.931 +306153 26.753 +306154 339.244 +306155 221.028 +306169 67.046 +306170 1.149 +306171 36.212 +306416 0.018 +306417 26.004 +306418 21.266 +306419 18.366 +306420 34.099 +306422 36.771 +306423 85.789 +306425 43.121 +306432 33.79 +306454 27.08 +306455 1.63 +306456 236.925 +306457 6.713 +306458 61.464 +306459 310.102 +306460 4.048 +306461 0.514 +306462 0.893 +306546 0.006 +306548 0.001 +306549 0.004 +306550 0.075 +306553 4.007 +306563 2.497 +306572 22.086 +306580 28.881 +306584 1.696 +306595 1.501 +306598 6.231 +306604 16.97 +306629 15.531 +306630 2.139 +306631 17.432 +306636 0.099 +306645 0.251 +306646 0.176 +306647 0.04 +306651 0.292 +306652 0.142 +306653 1.808 +306654 2.199 +306656 2.557 +306657 13.422 +306705 25.841 +306709 28.12 +306772 20.615 +306773 10.523 +306776 0.161 +306777 8.137 +306793 38.425 +306794 7.24 +306801 24.496 +306826 7.987 +306896 0.018 +306897 0.003 +306926 1.905 +306929 4.56 +306936 43.566 +307014 24.117 +307015 7.054 +307016 5.631 +307017 13.81 +307042 10.995 +307044 1.417 +307045 0.823 +307046 1.38 +307047 9.371 +307048 3.394 +307049 0.443 +307050 5.616 +307051 9.127 +307052 0.365 +307053 0.88 +307054 1.023 +307055 15.762 +307062 14.355 +307063 4.125 +307073 16.141 +307075 2.009 +307076 30.169 +307082 23.845 diff --git a/Alignment/APEEstimation/data/lumiperrun2018.txt b/Alignment/APEEstimation/data/lumiperrun2018.txt new file mode 100644 index 0000000000000..6ad5cbb91d1e6 --- /dev/null +++ b/Alignment/APEEstimation/data/lumiperrun2018.txt @@ -0,0 +1,551 @@ +314881 0 +315257 7.979 +315258 0.017 +315259 14.366 +315264 39.224 +315265 6.684 +315267 36.402 +315270 78.249 +315322 177.319 +315339 92.546 +315357 151.55 +315361 116.836 +315363 18.48 +315365 4.018 +315366 100.834 +315420 254.524 +315488 232.752 +315489 102.182 +315490 4.357 +315506 27.973 +315509 16.619 +315510 100.746 +315512 228.931 +315543 36.618 +315555 21.9 +315556 8.215 +315557 82.335 +315640 10.443 +315641 1.206 +315642 30.611 +315644 61.476 +315645 158.932 +315646 183.546 +315647 7.609 +315648 15.787 +315689 330.131 +315690 115.631 +315702 22.597 +315703 152.389 +315704 14.132 +315705 127.295 +315713 352.095 +315721 207.02 +315741 24.216 +315764 111.716 +315770 119.13 +315784 67.446 +315785 102.831 +315786 21.21 +315790 167.864 +315800 208.649 +315801 95.328 +315840 343.999 +315973 275.276 +315974 19.316 +316058 133.08 +316059 163.282 +316060 187.857 +316061 9.953 +316062 3.441 +316082 137.53 +316109 0.552 +316110 79.017 +316111 14.573 +316112 0 +316113 19.663 +316114 351.729 +316151 0.75 +316153 245.686 +316186 14.893 +316187 509.898 +316199 388.575 +316200 2.148 +316201 106.02 +316202 73.318 +316216 171.744 +316217 85.089 +316218 236.9 +316219 48.576 +316239 202.781 +316240 273.3 +316241 51.816 +316271 29.294 +316361 26.87 +316362 54.941 +316363 5.064 +316377 2.112 +316378 3.271 +316379 8.038 +316380 171.911 +316455 9.616 +316456 0.555 +316457 316.79 +316469 202.407 +316470 112.705 +316472 66.96 +316505 389.045 +316559 0 +316569 478.488 +316590 183.429 +316613 31.188 +316615 48.728 +316664 0.005 +316665 94.986 +316666 269.016 +316667 40.735 +316700 124.498 +316701 138.597 +316702 92.044 +316715 5.681 +316716 59.58 +316717 61.15 +316718 95.539 +316719 31.79 +316720 40.114 +316721 3.777 +316722 129.484 +316723 11.09 +316758 440.119 +316766 540.971 +316876 209.502 +316877 108.483 +316879 37.391 +316928 43.315 +316944 477.148 +316985 167.286 +316993 72.331 +316994 4.587 +316995 179.421 +317080 8.123 +317087 259.554 +317088 0.008 +317089 205.613 +317182 391.718 +317212 49.769 +317213 120.98 +317279 64.932 +317291 267.785 +317292 81.932 +317295 4.155 +317296 21.482 +317297 125.799 +317319 52.026 +317320 445.563 +317338 13.521 +317339 67.871 +317340 135.492 +317382 23.644 +317383 22.036 +317391 3.201 +317392 480.456 +317434 76.125 +317435 347.199 +317438 52.744 +317475 30.501 +317478 7.632 +317479 3.955 +317480 4.569 +317481 6.947 +317482 6.049 +317484 159.279 +317488 174.899 +317509 116.085 +317510 3.638 +317511 117 +317512 104.356 +317527 434.963 +317591 107.981 +317626 509.817 +317640 251.58 +317641 257.744 +317648 33.224 +317649 185.578 +317650 246.268 +317661 377.607 +317663 147.601 +317664 0 +317683 114.086 +317696 223.282 +318227 0 +318228 0 +318712 0 +318733 2.69 +318816 28.774 +318819 4.894 +318820 10.196 +318828 13.317 +318872 88.558 +318874 94.917 +318876 44.263 +318877 153.063 +318944 0.001 +319077 16.405 +319310 0.004 +319337 224.593 +319347 205.418 +319348 4.855 +319349 39.728 +319449 243.493 +319450 201.193 +319456 85.04 +319459 16.604 +319460 0 +319486 19.919 +319503 88.691 +319524 444.821 +319526 67.425 +319528 57.858 +319579 771.257 +319625 68.219 +319639 392.208 +319656 85.233 +319657 49.621 +319658 59.878 +319659 20.962 +319678 92.073 +319687 16.414 +319697 60.05 +319698 35.741 +319756 470.238 +319840 48.945 +319841 20.616 +319847 1.494 +319848 18.293 +319849 166.303 +319851 1.079 +319852 0.06 +319853 69.835 +319854 52.974 +319908 18.294 +319909 1.088 +319910 291.925 +319912 12.855 +319913 11.999 +319914 6.576 +319915 83.175 +319941 91.184 +319942 17.034 +319950 64.012 +319991 270.744 +319992 62.668 +319993 175.454 +320002 48.851 +320006 109.549 +320007 43.194 +320008 21.598 +320009 29.401 +320010 71.342 +320011 58.384 +320012 17.441 +320023 95.188 +320024 120.883 +320025 28.769 +320026 48.008 +320038 210.015 +320039 7.856 +320040 169.72 +320058 6.772 +320059 39.241 +320060 15.071 +320061 17.52 +320062 6.672 +320063 21.2 +320064 62.044 +320065 218.132 +320377 0 +320500 0.033 +320569 0.059 +320570 0.084 +320571 0.013 +320673 62.703 +320674 33.949 +320688 151.226 +320712 67.332 +320757 112.969 +320804 305.407 +320807 0.322 +320809 104.81 +320821 58.352 +320822 142.985 +320823 76.341 +320824 157.749 +320838 88.699 +320840 134.611 +320841 48.355 +320853 113.509 +320854 37.854 +320855 139.939 +320856 31.978 +320857 50.441 +320858 35.817 +320859 4.998 +320887 100.694 +320888 40.999 +320916 6.408 +320917 456.336 +320920 23.568 +320933 50.139 +320934 200.871 +320936 74.726 +320941 15.816 +320980 37.865 +320995 25.382 +320996 47.345 +321004 45.439 +321005 18.327 +321006 49.413 +321007 193.462 +321009 14.042 +321010 54.622 +321011 27.616 +321012 18.893 +321051 317.887 +321055 133.874 +321067 178.353 +321068 150.153 +321069 50.809 +321119 60.087 +321121 16.138 +321122 122.55 +321123 0.008 +321124 189.134 +321126 81.369 +321134 8.787 +321138 145.742 +321140 120.524 +321149 445.236 +321162 12.012 +321164 26.711 +321165 0.384 +321166 3.431 +321167 280.825 +321177 175.656 +321178 24.857 +321218 288.169 +321219 185.067 +321221 6.144 +321230 23.581 +321231 14.749 +321232 6.223 +321233 236.103 +321261 4.929 +321262 2.101 +321282 0 +321283 122.297 +321294 20.945 +321295 209.065 +321296 14.737 +321305 614.146 +321310 16.883 +321311 0.493 +321312 61.743 +321313 27.805 +321393 52.713 +321396 371.482 +321397 54.046 +321414 387.821 +321415 162.276 +321431 57.803 +321432 17.025 +321433 43.422 +321434 183.96 +321436 141.274 +321457 491.855 +321461 22.062 +321475 483.601 +321709 0.027 +321710 17.456 +321712 91.687 +321730 104.71 +321732 360.769 +321735 24.291 +321755 218.614 +321758 45.974 +321759 2.138 +321760 157.24 +321773 28.281 +321774 49.459 +321775 17.223 +321776 14.747 +321777 87.765 +321778 38.719 +321780 128.051 +321781 115.23 +321794 66.906 +321796 0.529 +321813 145.648 +321815 5.198 +321817 150.842 +321818 140.525 +321820 33.35 +321831 252.008 +321832 118.434 +321833 75.564 +321834 51.549 +321879 108.133 +321880 42.605 +321887 281.571 +321908 147.707 +321909 346.853 +321917 268.144 +321919 1.778 +321933 103.457 +321960 7.932 +321961 134.145 +321973 364.865 +321975 149.073 +321988 416.575 +321990 83.153 +322013 1.995 +322014 7.142 +322022 466.979 +322040 14.843 +322057 9.474 +322068 236.965 +322079 154.185 +322088 176.924 +322106 286.044 +322113 41.67 +322118 275.685 +322179 477.364 +322201 86.1 +322204 317.516 +322222 94.911 +322252 426.757 +322317 20.18 +322319 56.178 +322322 328.109 +322324 71.449 +322332 318.745 +322348 419.67 +322355 34.663 +322356 237.438 +322381 181.585 +322407 174.54 +322430 238.417 +322431 234.431 +322480 110.92 +322483 0 +322487 0 +322492 294.315 +322510 3.162 +322599 95.947 +322602 24.342 +322603 2.761 +322605 90.614 +322616 18.079 +322617 138.687 +322625 341.969 +322633 52.231 +323413 0 +323414 6.606 +323416 14.198 +323417 5.998 +323418 9.058 +323419 10.317 +323420 2.828 +323421 2.754 +323422 0.074 +323423 22.385 +323470 83.847 +323471 76.015 +323472 19.45 +323473 55.094 +323474 78.721 +323475 15.38 +323487 157.905 +323488 197.78 +323492 5.95 +323493 27.663 +323495 34.141 +323524 182.26 +323525 246.388 +323526 67.832 +323693 41.09 +323696 89.73 +323702 148.513 +323725 119.823 +323726 14.728 +323727 253.007 +323755 302.018 +323775 48.452 +323778 273.365 +323790 279.544 +323794 16.429 +323841 156.807 +323857 97.588 +323940 433.407 +323954 15.48 +323976 15.171 +323978 22.087 +323980 68.171 +323983 57.194 +323997 130.011 +324021 254.578 +324022 132.29 +324077 244.9 +324201 366.809 +324202 39.72 +324205 23.278 +324206 17.783 +324207 4.108 +324209 17.711 +324237 76.231 +324245 421.399 +324293 518.939 +324315 70.782 +324318 101.426 +324420 202.661 +324564 0.016 +324570 0 +324571 0.013 +324729 15.935 +324747 310.581 +324764 30.333 +324765 88.664 +324769 47.322 +324772 19.602 +324785 187.095 +324791 247.724 +324835 111.572 +324840 27.864 +324841 303.483 +324846 70.251 +324878 409.002 +324897 51.027 +324970 486.994 +324980 527.983 +324997 42.308 +324998 114.778 +324999 3.91 +325000 95.258 +325001 121.488 +325022 436.917 +325057 121.483 +325097 18.725 +325098 2.005 +325099 139.74 +325100 74.857 +325101 119.275 +325110 3.279 +325111 3.834 +325113 21.895 +325114 1.396 +325117 87.506 +325159 62.202 +325168 4.341 +325169 5.634 +325170 265.367 +325172 69.03 +325175 1.504 diff --git a/Alignment/APEEstimation/test/autoSubmitter/autoSubmitter.py b/Alignment/APEEstimation/test/autoSubmitter/autoSubmitter.py index b1551821edc08..80f23ec00ea4c 100644 --- a/Alignment/APEEstimation/test/autoSubmitter/autoSubmitter.py +++ b/Alignment/APEEstimation/test/autoSubmitter/autoSubmitter.py @@ -5,126 +5,29 @@ import sys import os import subprocess +import threading import shutil import time import re +import ROOT +from helpers import * sys.path.append("../plottingTools") shelve_name = "dump.shelve" # contains all the measurement objects and plot objects history_file = "history.log" clock_interval = 20 # in seconds -delete_logs_after_finish = True # if it is not desired to keep the log and submit script files - -# regex matching on key, replacement of groups on value -# implement any other shortcuts that you want to use -shortcuts = {} -#sources -shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db" -shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db" -shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db" -shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db" -shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS" - - -# Exact numbers don't really matter, but it is important that each one has a unique -# number, so that states are distinguishable -STATE_NONE = -1 -STATE_ITERATION_START=0 -STATE_BJOBS_WAITING=1 -STATE_BJOBS_DONE=2 -STATE_BJOBS_FAILED=12 -STATE_MERGE_WAITING=3 -STATE_MERGE_DONE=4 -STATE_MERGE_FAILED=14 -STATE_SUMMARY_WAITING=5 -STATE_SUMMARY_DONE=6 -STATE_SUMMARY_FAILED=16 -STATE_LOCAL_WAITING=7 -STATE_LOCAL_DONE=8 -STATE_LOCAL_FAILED=18 -STATE_FINISHED=9 - -status_map = {} -status_map[STATE_NONE] = "STATE_NONE" -status_map[STATE_ITERATION_START] = "STATE_ITERATION_START" -status_map[STATE_BJOBS_WAITING] = "STATE_BJOBS_WAITING" -status_map[STATE_BJOBS_DONE] = "STATE_BJOBS_DONE" -status_map[STATE_BJOBS_FAILED] = "STATE_BJOBS_FAILED" -status_map[STATE_MERGE_WAITING] = "STATE_MERGE_WAITING" -status_map[STATE_MERGE_DONE] = "STATE_MERGE_DONE" -status_map[STATE_MERGE_FAILED] = "STATE_MERGE_FAILED" -status_map[STATE_SUMMARY_WAITING] = "STATE_SUMMARY_WAITING" -status_map[STATE_SUMMARY_DONE] = "STATE_SUMMARY_DONE" -status_map[STATE_SUMMARY_FAILED] = "STATE_SUMMARY_FAILED" -status_map[STATE_LOCAL_WAITING] = "STATE_LOCAL_WAITING" -status_map[STATE_LOCAL_DONE] = "STATE_LOCAL_DONE" -status_map[STATE_LOCAL_FAILED] = "STATE_LOCAL_FAILED" -status_map[STATE_FINISHED] = "STATE_FINISHED" -base = "" - - -def ensurePathExists(path): - import os - import errno - try: - os.makedirs(path) - except OSError as exception: - if exception.errno != errno.EEXIST: - raise - - -def replaceAllRanges(string): - if "[" in string and "]" in string: - strings = [] - posS = string.find("[") - posE = string.find("]") - nums = string[posS+1:posE].split(",") - expression = string[posS:posE+1] - - nums = string[string.find("[")+1:string.find("]")] - for interval in nums.split(","): - interval = interval.strip() - if "-" in interval: - lowNum = int(interval.split("-")[0]) - upNum = int(interval.split("-")[1]) - for i in range(lowNum, upNum+1): - newstring = string[0:posS]+str(i)+string[posE+1:] - newstring = replaceAllRanges(newstring) - strings += newstring - else: - newstring = string[0:posS]+interval+string[posE+1:] - newstring = replaceAllRanges(newstring) - strings += newstring - return strings - else: - return [string,] - - -def replaceShortcuts(toScan): - global shortcuts - for key, value in shortcuts.items(): - match = re.search(key, toScan) - if match and match.group(0) == toScan: - return value.format(*match.groups()) - # no match - return toScan - -def loadConditions(dictionary): - hasAlignmentCondition = False - conditions = [] - for key, value in dictionary.items(): - if key.startswith("condition"): - record = key.split(" ")[1] - connect, tag = value.split(" ") - if record == "TrackerAlignmentRcd": - hasAlignmentCondition = True - conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag}) - return conditions, hasAlignmentCondition +delete_logs_after_finish = False # if it is not desired to keep the log and submit script files def save(name, object): - sh = shelve.open(shelve_name) - sh[name] = object - sh.close() + # in case of multiple threads running this stops potentially problematic file access + global lock + lock.acquire() + try: + sh = shelve.open(shelve_name) + sh[name] = object + sh.close() + finally: + lock.release() class Dataset: name = "" @@ -237,10 +140,10 @@ def __init__(self, name, dataset, alignment, config, additionalOptions={}): if self.resultPlotDo == "True": self.resultPlotTitle = self.resultPlotTitle.replace("~", " ") - # Adds new section to config file so it is read in the next step + # Adds new section to config file, result plots are loaded from the config in a later step sectionName = "resultplot:{}".format(self.name) config.add_section(sectionName) - config.set(sectionName, "wait {}".format(self.resultPlotLabel), "{} {}".format(self.name, min(14, self.maxIterations-1))) + config.set(sectionName, "wait {}".format(self.resultPlotLabel), "{} iteration={}".format(self.name, min(14, self.maxIterations-1))) config.set(sectionName, "title", self.resultPlotTitle) config.set(sectionName, "outPath", self.resultPlotOutPath) @@ -249,12 +152,13 @@ def get_status(self): def print_status(self): print("APE Measurement {} in iteration {} is now in status {}".format(self.name, self.curIteration, self.get_status())) - + + # submit jobs for track refit and hit categorization def submit_jobs(self): toSubmit = [] allConditions = self.alignment.conditions+self.dataset.conditions - allConditions = list({v['record']:v for v in allConditions}.values()) # should we clean for duplicate records? the overlap record last defined (from dataset) + allConditions = list({v['record']:v for v in allConditions}.values()) # should we clean for duplicate records? the record last defined (from dataset) # will be kept in case of overlap, which is the same as if there was no overlap removal # If conditions are made, create file to load them from @@ -326,7 +230,7 @@ def submit_jobs(self): cluster = subOut.split(" ")[-1][:-1] for i in range(self.dataset.nFiles): - # list contains condor log files from which to read when job is terminated + # list contains condor log files from which to read when job is terminated to detect errors self.runningJobs.append((logFileTemp.format(i), errorFileTemp.format(i), "{}.{}".format(cluster, i))) @@ -334,14 +238,15 @@ def submit_jobs(self): self.print_status() def check_jobs(self): - # Job was aborted by the user. lastStatus = self.status stillRunningJobs = [] # check all still running jobs for logName, errName, jobId in self.runningJobs: + # read condor logs instead of doing condor_q or similar, as it is much faster if not os.path.isfile(logName): print("{} does not exist even though it should, marking job as failed".format(logName)) self.failedJobs.append( (logName, errName) ) + break with open(logName, "r") as logFile: log = logFile.read() if not "submitted" in log: @@ -363,11 +268,11 @@ def check_jobs(self): print("Job {} of measurement {} in iteration {} finished successfully".format(jobId, self.name, self.curIteration)) else: # Fatal error in stderr - print("Job {} of measurement {} in iteration {} has a fatal error".format(jobId, self.name, self.curIteration)) + print("Job {} of measurement {} in iteration {} has a fatal error, check stderr".format(jobId, self.name, self.curIteration)) self.failedJobs.append( (logName, errName) ) else: # nonzero return value - print("Job {} of measurement {} in iteration {} failed".format(jobId, self.name, self.curIteration)) + print("Job {} of measurement {} in iteration {} failed, check stderr".format(jobId, self.name, self.curIteration)) self.failedJobs.append( (logName, errName) ) else: stillRunningJobs.append( (logName, errName, jobId) ) @@ -399,7 +304,8 @@ def check_jobs(self): if lastStatus != self.status: self.print_status() - + + # merges files from jobs def do_merge(self): self.status = STATE_MERGE_WAITING if self.alignment.isDesign: @@ -432,6 +338,7 @@ def do_merge(self): self.finishTime = subprocess.check_output(["date"]).strip() self.print_status() + # calculates APE def do_summary(self): self.status = STATE_SUMMARY_WAITING from autoSubmitterTemplates import summaryTemplate @@ -448,7 +355,8 @@ def do_summary(self): self.status = STATE_SUMMARY_FAILED self.finishTime = subprocess.check_output(["date"]).strip() self.print_status() - + + # saves APE to .db file so it can be read out next iteration def do_local_setting(self): self.status = STATE_LOCAL_WAITING from autoSubmitterTemplates import localSettingTemplate @@ -485,6 +393,74 @@ def purge(self): shutil.rmtree(folderName) # remove log-files as well? + def run_iteration(self): + global threadcounter + global measurements + threadcounter.acquire() + try: + if self.status == STATE_ITERATION_START: + # start bjobs + print("APE Measurement {} just started iteration {}".format(self.name, self.curIteration)) + while True: + try: + self.submit_jobs() + save("measurements", measurements) + break + except KeyboardInterrupt: + exit() + except: + # this is needed in case the scheduler goes down + print("Error submitting, waiting for 1 minute") + time.sleep(60) + + continue # no reason to immediately check jobs + if self.status == STATE_BJOBS_WAITING: + # check if bjobs are finished + self.check_jobs() + save("measurements", measurements) + if self.status == STATE_BJOBS_DONE: + # merge files + self.do_merge() + save("measurements", measurements) + if self.status == STATE_MERGE_DONE: + # start summary + self.do_summary() + save("measurements", measurements) + if self.status == STATE_SUMMARY_DONE: + # start local setting (only if not a baseline measurement) + if self.alignment.isDesign: + self.status = STATE_LOCAL_DONE + else: + self.do_local_setting() + save("measurements", measurements) + if self.status == STATE_LOCAL_DONE: + self.finish_iteration() + save("measurements", measurements) + # go to next iteration or finish measurement + if self.status == STATE_BJOBS_FAILED or \ + self.status == STATE_MERGE_FAILED or \ + self.status == STATE_SUMMARY_FAILED or \ + self.status == STATE_LOCAL_FAILED or \ + self.status == STATE_FINISHED: + with open(history_file, "a") as fi: + fi.write("APE measurement {name} which was started at {start} finished at {end} with state {state} in iteration {iteration}\n".format(name=self.name, start=self.startTime, end=self.finishTime, state=self.get_status(), iteration=self.curIteration)) + if self.status == STATE_FINISHED: + global finished_measurements + finished_measurements[self.name] = self + save("finished", finished_measurements) + else: + global failed_measurements + failed_measurements[self.name] = self + self.status = STATE_NONE + save("failed", failed_measurements) + save("measurements", measurements) + if self.status == STATE_ITERATION_START: # this ensures that jobs do not go into idle if many measurements are done simultaneously + # start bjobs + print("APE Measurement {} just started iteration {}".format(self.name, self.curIteration)) + self.submit_jobs() + save("measurements", measurements) + finally: + threadcounter.release() class ResultPlot: def __init__(self, config, name): @@ -498,18 +474,33 @@ def __init__(self, config, name): self.granularity = "standardGranularity" for key, value in rpDict.items(): + + if key.startswith("wait ") or key.startswith("load "): + optsList = value.split(" ") + val = optsList[0] + optsDict = {} + for i in range(1, len(optsList)): + k, v = optsList[i].split("=") + if k == "marker" or k == "color": + v = eval(v) # convert to int, makes it possible to use ROOT variables + optsDict[k] = v + + if key.startswith("wait "): label = key.split(" ")[1] label = label.replace("~", " ") - if len(value.split(" ")) > 1: - name, iteration = value.split(" ") + + if optsDict.has_key("iteration"): + iteration = optsDict["iteration"] + optsDict.pop("iteration") else: - name, iteration = value, "14" - self.waitingFor.append((name, iteration, label)) + iteration = "14" + + self.waitingFor.append((val, iteration, label, optsDict.copy())) elif key.startswith("load "): label = key.split(" ")[1] label = label.replace("~", " ") - self.loadingFrom.append((value,label)) + self.loadingFrom.append((val,label, optsDict.copy())) else: setattr(self, key, value) @@ -528,16 +519,15 @@ def do_plot(self): plotter.setOutputPath(self.outPath) plotter.setTitle(self.title) plotter.setGranularity(getattr(granularity, self.granularity)) - - for path, label in self.loadingFrom: - plotter.addInputFile(label, path, label) - for name, iteration, label in self.waitingFor: + for path, label, optsDict in self.loadingFrom: + plotter.addInputFile(label, path, **optsDict) + for name, iteration, label, optsDict in self.waitingFor: path = '{}/hists/{}/iter{}/allData_iterationApe.root'.format(base, name, iteration) - plotter.addInputFile(name, path, label) + plotter.addInputFile(label, path, **optsDict) ensurePathExists(self.outPath) plotter.draw() - - + + def main(): parser = argparse.ArgumentParser(description="Automatically run APE measurements") parser.add_argument("-c", "--config", action="append", dest="configs", default=[], @@ -550,16 +540,24 @@ def main(): help="Resume interrupted APE measurements which are stored in shelves (specify shelves)") parser.add_argument("-d", "--dump", action="store", dest="dump", default=None, help='Specify in which .shelve file to store the measurements and plots') + parser.add_argument("-n", "--ncores", action="store", dest="ncores", default=1, type=int, + help='Number of threads running in parallel') args = parser.parse_args() global base global clock_interval global shelve_name + global threadcounter + global lock + + threadcounter = threading.BoundedSemaphore(args.ncores) + lock = threading.Lock() if args.dump != None: # choose different file than default shelve_name = args.dump - + elif args.resume != []: + shelve_name = args.resume[0] try: base = os.environ['CMSSW_BASE']+"/src/Alignment/APEEstimation" except KeyError: @@ -574,8 +572,12 @@ def main(): for toConvert in args.purge: purgeTargets += replaceAllRanges(toConvert) + global measurements measurements = [] + global finished_measurements finished_measurements = {} + global failed_measurements + failed_measurements = {} resultPlots = [] if args.resume != []: @@ -584,7 +586,11 @@ def main(): sh = shelve.open(resumeFile) resumed = sh["measurements"] resumed_plots = sh["resultPlots"] + + resumed_failed = sh["failed"] + resumed_finished = sh["finished"] sh.close() + for res in resumed: measurements.append(res) print("Measurement {} in state {} in iteration {} was resumed".format(res.name, res.get_status(), res.curIteration)) @@ -599,6 +605,10 @@ def main(): for res in resumed_plots: resultPlots.append(res) print("Result plot {} was resumed".format(res.name)) + + failed_measurements.update(resumed_failed) + finished_measurements.update(resumed_finished) + except IOError: print("Could not resume because {} could not be opened, exiting".format(shelve_name)) sys.exit() @@ -628,8 +638,7 @@ def main(): for i in range(2, len(settings)): setting = settings[i].strip() - key = setting.split("=")[0] - value = setting.split("=")[1] + key, value = setting.split("=") additionalOptions[key] = value measurement = ApeMeasurement(name, dataset, alignment, config, additionalOptions) @@ -649,57 +658,19 @@ def main(): measurements = [measurement for measurement in measurements if not (measurement.status==STATE_NONE or measurement.status == STATE_FINISHED)] save("measurements", measurements) save("resultPlots", resultPlots) + save("failed", failed_measurements) + save("finished", finished_measurements) + list_threads = [] for measurement in measurements: - if measurement.status == STATE_ITERATION_START: - # start bjobs - print("APE Measurement {} just started iteration {}".format(measurement.name, measurement.curIteration)) - measurement.submit_jobs() - save("measurements", measurements) - continue # no reason to immediately check jobs - if measurement.status == STATE_BJOBS_WAITING: - # check if bjobs are finished - measurement.check_jobs() - save("measurements", measurements) - if measurement.status == STATE_BJOBS_DONE: - # merge files - measurement.do_merge() - save("measurements", measurements) - if measurement.status == STATE_MERGE_DONE: - # start summary - measurement.do_summary() - save("measurements", measurements) - if measurement.status == STATE_SUMMARY_DONE: - # start local setting (only if not a baseline measurement) - if measurement.alignment.isDesign: - measurement.status = STATE_LOCAL_DONE - else: - measurement.do_local_setting() - save("measurements", measurements) - if measurement.status == STATE_LOCAL_DONE: - measurement.finish_iteration() - save("measurements", measurements) - # go to next iteration or finish measurement - if measurement.status == STATE_BJOBS_FAILED or \ - measurement.status == STATE_MERGE_FAILED or \ - measurement.status == STATE_SUMMARY_FAILED or \ - measurement.status == STATE_LOCAL_FAILED or \ - measurement.status == STATE_FINISHED: - with open(history_file, "a") as fi: - fi.write("APE measurement {name} which was started at {start} finished at {end} with state {state} in iteration {iteration}\n".format(name=measurement.name, start=measurement.startTime, end=measurement.finishTime, state=measurement.get_status(), iteration=measurement.curIteration)) - - if measurement.status == STATE_FINISHED: - finished_measurements[measurement.name] = measurement - else: - measurement.status = STATE_NONE - - - save("measurements", measurements) - if measurement.status == STATE_ITERATION_START: # this ensures that jobs do not go into idle if many measurements are done simultaneously - # start bjobs - print("APE Measurement {} just started iteration {}".format(measurement.name, measurement.curIteration)) - measurement.submit_jobs() - save("measurements", measurements) + t = threading.Thread(target=measurement.run_iteration) + list_threads.append(t) + t.start() + + # wait for iterations to finish + for t in list_threads: + t.join() + # Check if there are plots to do changed = False diff --git a/Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py b/Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py index c5f52bf8aa18a..499bf4de20012 100644 --- a/Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py +++ b/Alignment/APEEstimation/test/autoSubmitter/autoSubmitterTemplates.py @@ -6,7 +6,7 @@ eval `scramv1 runtime -csh` -source /afs/cern.ch/cms/caf/setup.csh +#~ source /afs/cern.ch/cms/caf/setup.csh cd $curDir xrdcp {inputFile} reco.root @@ -25,11 +25,27 @@ request_memory = 2000M request_disk = 400M batch_name = {jobName} -+JobFlavour = "longlunch" ++JobFlavour = "workday" Queue Arguments from ( {arguments}) """ +# Use this one only if connected to lxplus-t0.cern.ch +#~ condorSubTemplate=""" +#~ Executable = {jobFile} +#~ Universe = vanilla +#~ Output = {outputFile} +#~ Error = {errorFile} +#~ Log = {logFile} +#~ request_memory = 2000M +#~ request_disk = 400M +#~ batch_name = {jobName} +#~ +JobFlavour = "workday" +#~ +AccountingGroup = "group_u_CMS.CAF.ALCA" +#~ Queue Arguments from ( +#~ {arguments}) +#~ """ + condorArgumentTemplate="""{fileNumber} {inputFile} """ diff --git a/Alignment/APEEstimation/test/autoSubmitter/config.ini b/Alignment/APEEstimation/test/autoSubmitter/config.ini index 21c8a5f5dfd21..8810a277238bb 100644 --- a/Alignment/APEEstimation/test/autoSubmitter/config.ini +++ b/Alignment/APEEstimation/test/autoSubmitter/config.ini @@ -15,7 +15,7 @@ isMC=False # in apeEstimation_cfg.py. If it is not defined or None, it will not overwrite # anything defined in apeEstimation_cfg.py globalTag=None -# name as defined in apeEstimation_cfg.py +# name as defined in apeEstimation_cfg.py, optional if alignment is loaded by condition alignmentName=alignmentObjectName # name of baseline folder. Only used if isDesign is False, elsewise the # name of the APE measurement is used, Design by default @@ -26,6 +26,10 @@ isDesign=False # it will override the one defined by alignmentName condition TrackerAlignmentRcd=sqlite_file:/afs/asdf.db Alignments condition TrackerSurfaceDeformationRcd=sqlite_file:/afs/asdf.db Deformations +# another way to load these is: +condition Alignments+Deformations:sqlite_file:/afs/asdf.db +# in this case, the Rcd name has to be guessed by the autoSubmitter, +# so functionality is limited # define measurements like this [measurements] @@ -50,6 +54,8 @@ exampleName: exampleDataset alignmentObject wait label:exampleName # load APE from these files (in labels ~ is replaced with whitespaces) load label~with~empty~spaces:/path/to/allData_iterationApe.root +# optional attributes +load label~with~empty~spaces2:/path/to/allData_iterationApe.root color=ROOT.kRed+2 marker=0 # output path. If not defined, store in hists/Name outPath=/asdf/ # title to the plot, optional diff --git a/Alignment/APEEstimation/test/autoSubmitter/helpers.py b/Alignment/APEEstimation/test/autoSubmitter/helpers.py new file mode 100644 index 0000000000000..136095a8cfc77 --- /dev/null +++ b/Alignment/APEEstimation/test/autoSubmitter/helpers.py @@ -0,0 +1,135 @@ +from __future__ import print_function +import re +import os +import errno +shortcuts = {} + +# regex matching on key, replacement of groups on value +# implement any other shortcuts that you want to use +#sources +shortcuts["mp([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm/alignments_MP.db" +shortcuts["mp([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp{0}/jobData/jobm{1}/alignments_MP.db" +shortcuts["sm([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/sm{0}/alignments_iter{1}.db" +shortcuts["um([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm/um{0}.db" +shortcuts["um([0-9]*)_jobm([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/um{0}/jobData/jobm{1}/um{0}.db" +shortcuts["hp([0-9]*)_iter([0-9]*)"] = "sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HipPy/alignments/hp{0}/alignments_iter{1}.db" +shortcuts["prod"] = "frontier://FrontierProd/CMS_CONDITIONS" + +# Exact numbers don't really matter, but it is important that each one has a unique +# number, so that states are distinguishable +STATE_NONE = -1 +STATE_ITERATION_START=0 +STATE_BJOBS_WAITING=1 +STATE_BJOBS_DONE=2 +STATE_BJOBS_FAILED=12 +STATE_MERGE_WAITING=3 +STATE_MERGE_DONE=4 +STATE_MERGE_FAILED=14 +STATE_SUMMARY_WAITING=5 +STATE_SUMMARY_DONE=6 +STATE_SUMMARY_FAILED=16 +STATE_LOCAL_WAITING=7 +STATE_LOCAL_DONE=8 +STATE_LOCAL_FAILED=18 +STATE_FINISHED=9 + +status_map = {} +status_map[STATE_NONE] = "none" +status_map[STATE_ITERATION_START] = "starting iteration" +status_map[STATE_BJOBS_WAITING] = "waiting for jobs" +status_map[STATE_BJOBS_DONE] = "jobs finished" +status_map[STATE_BJOBS_FAILED] = "jobs failed" +status_map[STATE_MERGE_WAITING] = "waiting for merging" +status_map[STATE_MERGE_DONE] = "merging done" +status_map[STATE_MERGE_FAILED] = "merging failed" +status_map[STATE_SUMMARY_WAITING] = "waiting for APE determination" +status_map[STATE_SUMMARY_DONE] = "APE determination done" +status_map[STATE_SUMMARY_FAILED] = "APE determination failed" +status_map[STATE_LOCAL_WAITING] = "waiting for APE saving" +status_map[STATE_LOCAL_DONE] = "APE saving done" +status_map[STATE_LOCAL_FAILED] = "APE saving failed" +status_map[STATE_FINISHED] = "finished" + +records = {} +records["Alignments"] = "TrackerAlignmentRcd" +records["TrackerAlignment"] = "TrackerAlignmentRcd" +records["Deformations"] = "TrackerSurfaceDeformationRcd" +records["TrackerSurfaceDeformations"] = "TrackerSurfaceDeformationRcd" +records["SiPixelTemplateDBObject"] = "SiPixelTemplateDBObjectRcd" +records["BeamSpotObjects"] = "BeamSpotObjectsRcd" + + +def ensurePathExists(path): + try: + os.makedirs(path) + except OSError as exception: + if exception.errno != errno.EEXIST: + raise + + +def replaceAllRanges(string): + if "[" in string and "]" in string: + strings = [] + posS = string.find("[") + posE = string.find("]") + nums = string[posS+1:posE].split(",") + expression = string[posS:posE+1] + + nums = string[string.find("[")+1:string.find("]")] + for interval in nums.split(","): + interval = interval.strip() + if "-" in interval: + lowNum = int(interval.split("-")[0]) + upNum = int(interval.split("-")[1]) + for i in range(lowNum, upNum+1): + newstring = string[0:posS]+str(i)+string[posE+1:] + newstring = replaceAllRanges(newstring) + strings += newstring + else: + newstring = string[0:posS]+interval+string[posE+1:] + newstring = replaceAllRanges(newstring) + strings += newstring + return strings + else: + return [string,] + + +def replaceShortcuts(toScan): + global shortcuts + for key, value in shortcuts.items(): + match = re.search(key, toScan) + if match and match.group(0) == toScan: + return value.format(*match.groups()) + # no match + return toScan + +def loadConditions(dictionary): + hasAlignmentCondition = False + conditions = [] + for key, value in dictionary.items(): + if key.startswith("condition"): + if len(value.split(" ")) > 1: # structure is "condition rcd:source tag" + record = key.split(" ")[1] + connect, tag = value.split(" ") + if record == "TrackerAlignmentRcd": + hasAlignmentCondition = True + conditions.append({"record":record, "connect":replaceShortcuts(connect), "tag":tag}) + else: + # structure is "condition tag:source", so we have to guess rcd from the tag. might also be "condition tag1+tag2+...+tagN:source" + global records + connect = value.strip() + tags = key.split(" ")[1] + for tag in tags.split("+"): + foundTag = False + for possibleTag, possibleRcd in records.items(): + if tag.startswith(possibleTag): + conditions.append({"record":possibleRcd, "connect":replaceShortcuts(connect), "tag":tag}) + if possibleRcd == "TrackerAlignmentRcd": + hasAlignmentCondition = True + foundTag = True + break + if not foundTag: + print("Did not find a record corresponding to {} tag".format(tag)) + exit() + + return conditions, hasAlignmentCondition diff --git a/Alignment/APEEstimation/test/batch/startSkim.py b/Alignment/APEEstimation/test/batch/startSkim.py index dd1d83e6c9dd4..a6c902b3b472f 100644 --- a/Alignment/APEEstimation/test/batch/startSkim.py +++ b/Alignment/APEEstimation/test/batch/startSkim.py @@ -121,7 +121,7 @@ def main(argv): help="Name of sample as defined in skimProducer_cfg.py. Multiple inputs possible") parser.add_argument("-c", "--consecutive", action="store_true", dest="consecutive", default=False, help="Do consecutive instead of parallel skims") - parser.add_argument("-n", "--ncores", action="store", dest="ncores", default=-1, type="int", + parser.add_argument("-n", "--ncores", action="store", dest="ncores", default=-1, type=int, help="Set maximum number of parallel skims to run") args = parser.parse_args() @@ -145,6 +145,8 @@ def main(argv): doSkim(sample) else: try: + # In a later PR, this should be migrated to condor to avoid + # overloading lxplus and transfer load to lxbatch pool = mp.Pool(args.ncores) pool.map_async(doSkim, args.samples) pool.close() diff --git a/Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py b/Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py index a696d1ed453a3..9439ed0afe089 100644 --- a/Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py +++ b/Alignment/APEEstimation/test/cfgTemplate/apeEstimator_cfg.py @@ -169,93 +169,16 @@ if options.alignRcd=='fromConditions': pass # Alignment is read from the conditions file in this case elif options.alignRcd=='design': - CondDBAlignment = CondDB.clone(connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS')) - process.myTrackerAlignment = cms.ESSource("PoolDBESSource", - CondDBAlignment, - timetype = cms.string("runnumber"), - toGet = cms.VPSet( - cms.PSet( - record = cms.string('TrackerAlignmentRcd'), - tag = cms.string('TrackerAlignment_Upgrade2017_design_v3') - ) - ) - ) - process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment") - - -elif options.alignRcd == 'misalTest': - CondDBAlignment = CondDB.clone(connect = cms.string('frontier://FrontierProd/CMS_CONDITIONS')) - process.myTrackerAlignment = cms.ESSource("PoolDBESSource", - CondDBAlignment, - timetype = cms.string("runnumber"), - toGet = cms.VPSet( - cms.PSet( - record = cms.string('TrackerAlignmentRcd'), - tag = cms.string('TrackerAlignment_Phase1Realignment_CRUZET_2M'), - ) - ) - ) - process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment") - -elif options.alignRcd == 'mp2705': - CondDBAlignment = CondDB.clone(connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp2705/jobData/jobm/alignments_MP.db')) - process.myTrackerAlignment = cms.ESSource("PoolDBESSource", - CondDBAlignment, - timetype = cms.string("runnumber"), - toGet = cms.VPSet( - cms.PSet( - record = cms.string('TrackerAlignmentRcd'), - tag = cms.string('Alignments'), - ) - ) - ) - process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment") - -elif options.alignRcd == 'mp2853': - CondDBAlignment = CondDB.clone() - process.myTrackerAlignment = cms.ESSource("PoolDBESSource", - CondDBAlignment, - timetype = cms.string("runnumber"), - toGet = cms.VPSet( - cms.PSet( - connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/mp2853/jobData/jobm3/alignments_MP.db'), - record = cms.string('TrackerAlignmentRcd'), - tag = cms.string('Alignments'), - ), - #~ cms.PSet( - #~ connect=cms.string('frontier://FrontierProd/CMS_CONDITIONS'), - #~ record=cms.string('SiPixelTemplateDBObjectRcd'), - #~ tag=cms.string('SiPixelTemplateDBObject_38T_TempForAlignmentReReco2018_v3'), - #~ ) - ) - ) - process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment") - -elif options.alignRcd == 'hp1370': - CondDBAlignment = CondDB.clone(connect = cms.string('sqlite_file:/afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN2/HIP/xiaomeng/CMSSW_7_4_6_patch5/src/Alignment/HIPAlignmentAlgorithm/hp1370/alignments.db')) - process.myTrackerAlignment = cms.ESSource("PoolDBESSource", - CondDBAlignment, - timetype = cms.string("runnumber"), - toGet = cms.VPSet( - cms.PSet( - record = cms.string('TrackerAlignmentRcd'), - tag = cms.string('Alignments'), - ) - ) - ) - process.es_prefer_trackerAlignment = cms.ESPrefer("PoolDBESSource","myTrackerAlignment") - - - + pass elif options.alignRcd == 'globalTag': - pass + pass elif options.alignRcd == 'useStartGlobalTagForAllConditions': - pass + pass elif options.alignRcd == '': - pass + pass else: - print('ERROR --- incorrect alignment: ', options.alignRcd) - exit(8888) + print('ERROR --- incorrect alignment: ', options.alignRcd) + exit(8888) ## APE if options.iterNumber!=0: diff --git a/Alignment/APEEstimation/test/plottingTools/drawIterations.py b/Alignment/APEEstimation/test/plottingTools/drawIterations.py index c45903f8443ff..8fa4d98a35c07 100644 --- a/Alignment/APEEstimation/test/plottingTools/drawIterations.py +++ b/Alignment/APEEstimation/test/plottingTools/drawIterations.py @@ -1,8 +1,8 @@ # Implementation to draw all iterations of an APE measurement # to check convergence - from iterationsPlotter import * from granularity import * +import os try: base = os.environ['CMSSW_BASE']+"/src/Alignment/APEEstimation" @@ -10,8 +10,8 @@ base = "" plot = IterationsPlotter() -plot.setOutputPath(base+"/workingArea/") -plot.setInputFile(base+"/workingArea/iter14/allData_iterationApe.root") -plot.setTitle("") +plot.setOutputPath(base+"/hists/iterations/") +plot.setInputFile(base+"/src/Alignment/APEEstimation/hists/workingArea/iter14/allData_iterationApe.root") +plot.setTitle("Title") plot.setGranularity(standardGranularity) plot.draw() diff --git a/Alignment/APEEstimation/test/plottingTools/drawResults.py b/Alignment/APEEstimation/test/plottingTools/drawResults.py index 3fa306931d2d4..c068c95df8fad 100644 --- a/Alignment/APEEstimation/test/plottingTools/drawResults.py +++ b/Alignment/APEEstimation/test/plottingTools/drawResults.py @@ -1,21 +1,18 @@ # Implementation to draw all iterations of an APE measurement # to check convergence - -import ROOT from resultPlotter import * -from systematics import * +from systematicErrors import * from granularity import * - +import os try: base = os.environ['CMSSW_BASE']+"/src/Alignment/APEEstimation" except KeyError: base = "" plot = ResultPlotter() -plot.setOutputPath(base+"/workingArea/") -# internal name (used for example when adding systematic errors), path to file, label, color (optional) -plot.addInputFile("placeholder1", base+"/workingArea/iter14/allData_iterationApe.root", "measurement A") -plot.addInputFile("placeholder2", base+"/workingArea2/iter14/allData_iterationApe.root", "measurement B", ROOT.kRed) -plot.setTitle("") +plot.setOutputPath(base+"/hists/workingArea/") +# label(also used as name when adding systematic errors), inputFile, color (optional, automatic by default), +# marker (optional, 20 by default, 0 is line), hitNumbers (optional, file number of hits in each sector, allData.root) +plot.addInputFile("label", "base/hists/workingArea/iter14/allData_iterationApe.root", color = ROOT.kGray+2) plot.setGranularity(standardGranularity) plot.draw() diff --git a/Alignment/APEEstimation/test/plottingTools/drawTrend.py b/Alignment/APEEstimation/test/plottingTools/drawTrend.py new file mode 100644 index 0000000000000..3ee6b0234248a --- /dev/null +++ b/Alignment/APEEstimation/test/plottingTools/drawTrend.py @@ -0,0 +1,37 @@ +# Implementation to draw APE trends +# to check convergence + +import ROOT +from trendPlotter import * +from granularity import * +import os + +try: + base = os.environ['CMSSW_BASE'] +except KeyError: + base = "../../../../.." + +plot = TrendPlotter() +plot.setOutputPath(base+"/src/Alignment/APEEstimation/trends/") +plot.setTitle("Title") +plot.setGranularity(standardGranularity) + +# The x-axis range is chosen depending on which year's APE are drawn. +# One can draw APE trends over multiple years + +# List of tuples for one trend +trendList = [] +# The last run is for example the first run of the next IOV minus 1 +#~ trendsList.append( (firstRun, lastRun, inputFile) ) + +# label, trendList, color (optional, automatic), marker (optional, 0) +# other options: dashed(optional, false) +plot.addTrend("label", trendList, color = ROOT.kBlack, marker=0) + +# if this is set to false, the plot is made with runs on the x-axis +plot.doLumi = True + +# if this is set to true, a logarithmic y-axis is used +plot.log = False + +plot.draw() diff --git a/Alignment/APEEstimation/test/plottingTools/resultPlotter.py b/Alignment/APEEstimation/test/plottingTools/resultPlotter.py index 1be503f44f2b4..b526319625a98 100644 --- a/Alignment/APEEstimation/test/plottingTools/resultPlotter.py +++ b/Alignment/APEEstimation/test/plottingTools/resultPlotter.py @@ -9,22 +9,26 @@ class ResultPlotter: def __init__(self): setTDRStyle() - self.names = {} self.inFiles = {} + self.hitNumbers = {} self.labels = {} self.colors = {} + self.markers = {} self.outPath = None self.hasSystematics = {} self.systematics = {} self.granularity = standardGranularity self.title = "" + self.order = [] - def addInputFile(self, name, inFile, label, color=None): - self.names[label] = name + def addInputFile(self, label, inFile, color=None, marker=20, hitNumbers=None): + self.order.append(label) self.inFiles[label] = inFile self.labels[label] = label - self.systematics[name] = [] - self.hasSystematics[name] = False + self.systematics[label] = [] + self.hasSystematics[label] = False + self.markers[label] = marker + self.hitNumbers[label] = hitNumbers if color != None: self.colors[label] = color else: @@ -43,14 +47,14 @@ def setOutputPath(self, outPath): def setTitle(self, title): self.title = title - def doSystematics(self, name): - self.hasSystematics[name] = True + def doSystematics(self, label): + self.hasSystematics[label] = True - def addSystematics(self, name, systematics, additive=True): - self.hasSystematics[name] = True + def addSystematics(self, label, systematics, additive=True): + self.hasSystematics[label] = True if not additive: - self.systematics[name] = [] - self.systematics[name].append(systematics) + self.systematics[label] = [] + self.systematics[label].append(systematics) def convertName(self, name): out = name.replace("Bpix", "BPIX") @@ -70,21 +74,47 @@ def convertName(self, name): out = out.replace("Out", "o") return out - def makeHist(self, name, sectorRange, coordinate, number): + def makeHitNumbers(self, label, sectorRange, coordinate): + self.numHitCounters += 1 sectors = range(sectorRange[0],sectorRange[1]+1) numSectors = len(sectors) - hist = ROOT.TH1F("{}hist{}_{}".format(name, number, coordinate), "", numSectors, 0, numSectors) + fi = ROOT.TFile(self.hitNumbers[label], "READ") + align = 22 + size = 0.02 + font = 42 + labels = [] + for i,sector in enumerate(sectors): + hitHist = fi.Get("ApeEstimator1/Sector_{}/Results/h_NorRes{}".format(sector, coordinate.upper())) + num = hitHist.GetEntries() + posX = (float(i)+0.5)/numSectors*(1-ROOT.gPad.GetLeftMargin()-ROOT.gPad.GetRightMargin())+ROOT.gPad.GetLeftMargin() + posY = (1-ROOT.gPad.GetTopMargin()-size)-1.2*size*self.numHitCounters + + label = ROOT.TLatex(posX, posY, "%.2E"%(num)) + label.SetNDC(True) + label.SetTextColor(self.colors[label]) + label.SetTextSize(size) + label.SetTextFont(font) + label.SetTextAngle(45) + label.SetTextAlign(align) + labels.append(label) + return labels + + def makeHist(self, label, sectorRange, coordinate, number): + sectors = range(sectorRange[0],sectorRange[1]+1) + numSectors = len(sectors) + + hist = ROOT.TH1F("{}hist{}_{}".format(label, number, coordinate), "", numSectors, 0, numSectors) hist.SetTitle(";;#sigma_{align," + coordinate.lower() + "} [#mum]") hist.SetAxisRange(0.,100.,"Y") syst = None - if self.hasSystematics[name]: + if self.hasSystematics[label]: syst = ROOT.TGraphAsymmErrors() - fi = ROOT.TFile(self.inFiles[name], "READ") + fi = ROOT.TFile(self.inFiles[label], "READ") nameTree = fi.Get("nameTree") - apeTree = fi.Get("iterTree{}".format(coordinate)) + apeTree = fi.Get("iterTree{}".format(coordinate.upper())) # Get last entries in branches (for apeTree) to retrieve result of this iteration # in iterTreeX/Y, the results of the previous iterations are also stored nameTree.GetEntry(0) @@ -93,15 +123,16 @@ def makeHist(self, name, sectorRange, coordinate, number): for sector in sectors: sectorApe = 10000. * (float(getattr(apeTree, "Ape_Sector_{}".format(sector))))**0.5 sectorName = self.convertName(str(getattr(nameTree, "Ape_Sector_{}".format(sector)))) + binWidth = hist.GetXaxis().GetBinCenter(iBin) - hist.GetXaxis().GetBinLowEdge(iBin) hist.SetBinContent(iBin, sectorApe) hist.SetBinError(iBin, 0.0000001) hist.GetXaxis().SetBinLabel(iBin, sectorName) - if self.hasSystematics[name]: + if self.hasSystematics[label]: sysErrUp = 0 sysErrDn = 0 # add up errors quadratically - for partError in self.systematics[name]: + for partError in self.systematics[label]: scaleFac = 1.0 if partError.isRelative[sector-1]: scaleFac = sectorApe @@ -115,7 +146,7 @@ def makeHist(self, name, sectorRange, coordinate, number): sysErrUp += (scaleFac*partError[coordinate][sector-1])**2 sysErrUp = sysErrUp**0.5 sysErrDn = sysErrDn**0.5 - binWidth = hist.GetXaxis().GetBinCenter(iBin) - hist.GetXaxis().GetBinLowEdge(iBin) + syst.SetPoint(iBin, hist.GetXaxis().GetBinCenter(iBin), sectorApe) syst.SetPointError(iBin, binWidth, binWidth, sysErrDn, sysErrUp) @@ -125,6 +156,7 @@ def makeHist(self, name, sectorRange, coordinate, number): return hist, syst def draw(self): + allLabels = [] for coordinate in self.granularity.sectors.keys(): plotNumber = 0 rangeList = self.granularity.sectors[coordinate] @@ -132,7 +164,7 @@ def draw(self): self.canvas = ROOT.TCanvas("canvas", "canvas", int(ROOT.gStyle.GetCanvasDefW()*len(range(sectorRange[0],sectorRange[1]+1))/10.),ROOT.gStyle.GetCanvasDefH()) ROOT.gPad.SetRightMargin(0.10) - legend = ROOT.TLegend(0.2,0.65,0.5,0.85) + legend = ROOT.TLegend(0.2,0.62,0.5,0.82) legend.SetFillColor(0) legend.SetFillStyle(0) legend.SetTextSize(0.04) @@ -142,24 +174,45 @@ def draw(self): firstHist = True histos = [] # need to save histos or they will be deleted right after variable is set to something else systGraphs = [] # same for systematics errors - for name in self.inFiles.keys(): + self.numHitCounters = 0 + for name in self.order: if firstHist: - drawMode = "E0" + addDraw = "" firstHist = False else: - drawMode = "E0same" + addDraw = "same" + + if self.markers[name] != 0: + drawMode = "P0%s"%(addDraw) + else: + drawMode = "hist%s"%(addDraw) + histo, syst = self.makeHist(name, sectorRange, coordinate, plotNumber) histo.SetMarkerColor(self.colors[name]) + histo.SetMarkerStyle(self.markers[name]) + if self.markers[name] == 0: + histo.SetMarkerSize(0) + histo.SetLineColor(self.colors[name]) + histo.SetLineWidth(2) + histo.Draw(drawMode) histos.append(histo) - legend.AddEntry(histo, self.labels[name], "p") - + if self.markers[name] != 0: + legend.AddEntry(histo, self.labels[name], "p") + else: + legend.AddEntry(histo, self.labels[name], "l") + if self.hasSystematics[name]: syst.SetFillColor(self.colors[name]) syst.SetFillStyle(3354) syst.Draw("02same") systGraphs.append(syst) + if self.hitNumbers[name] != None: + labels = self.makeHitNumbers(name, sectorRange, coordinate) + allLabels.extend(labels) + for label in labels: + label.Draw("same") legend.Draw() self.canvas.Update() @@ -172,7 +225,7 @@ def draw(self): if not os.path.isdir(self.outPath): os.makedirs(self.outPath) - self.canvas.SaveAs("{}/result_{}_{}.pdf".format(self.outPath, coordinate, self.granularity.names[coordinate][plotNumber])) + self.canvas.SaveAs("{}/results_{}_{}.pdf".format(self.outPath, coordinate, self.granularity.names[coordinate][plotNumber])) self.canvas = None legend = None histos = None diff --git a/Alignment/APEEstimation/test/plottingTools/trendPlotter.py b/Alignment/APEEstimation/test/plottingTools/trendPlotter.py new file mode 100644 index 0000000000000..d86fed8d5fc01 --- /dev/null +++ b/Alignment/APEEstimation/test/plottingTools/trendPlotter.py @@ -0,0 +1,307 @@ +import ROOT +ROOT.gROOT.SetBatch(True) +from setTDRStyle import setTDRStyle +from granularity import * +import os + +try: + base = os.environ['CMSSW_BASE'] +except KeyError: + base = "../../../../.." + +# 2016 is missing here +lumiFiles = { + 2017: "{base}/src/Alignment/APEEstimation/data/lumiperrun2017.txt", + 2018: "{base}/src/Alignment/APEEstimation/data/lumiperrun2018.txt", +} +pixelIOVs = { + 2016: [], + 2017: [297281, 298653, 299443, 300389, 301046, 302131, 303790, 303998, 304911], + 2018: [316758, 317527, 317661, 317664, 318227, 320377], +} + + +lumis = {} +intLumis = {} +runs = {} +years = [] + +for year in lumiFiles.keys(): + runs[year] = [] + years.append(year) + intLumis[year] = 0 + with open(lumiFiles[year].format(base=base), "r") as fi: + for line in fi: + run, lumi = line.split(" ") + run = int(run) + lumi = float(lumi)*0.001 # convert to fb^-1 + lumis[run] = lumi + runs[year].append(run) + intLumis[year] += lumi +years.sort() + +def runToLumi(whichRun, fromYear, inclusive=False): + lumiResult = 0.0 + for year in years: + if year < fromYear: + continue + for run in runs[year]: + if run > whichRun: + break + if run < whichRun: + lumiResult += lumis[run] + if run == whichRun and inclusive: + lumiResult += lumis[run] + return lumiResult + + +def whichYear(run): + thisYear = -1 + years = runs.keys() + years.sort() + for year in years: + if min(runs[year]) <= run: + thisYear = year + return thisYear + print "Run %d not in range of any year"%(run) + return -1 + + +class TrendPlotter: + def __init__(self): + setTDRStyle() + self.names = {} + self.outPath = None + self.granularity = standardGranularity + self.title = "" + self.points = [] + self.years = [] + self.doLumi = True + self.colors = [] + self.log = False + + def addTrend(self, label, points, dashed=False, color=None, marker=None): + self.points.append( (label, points, dashed, color, marker) ) + if color: + self.colors.append(color) + + def setGranularity(self, granularity): + self.granularity = granularity + + def setOutputPath(self, outPath): + self.outPath = outPath + + def setTitle(self, title): + self.title = title + + def setLog(self, log=True): + self.log = log + + def convertName(self, name): + out = name.replace("Bpix", "BPIX") + out = out.replace("Fpix", "FPIX") + out = out.replace("Plus", "+") + out = out.replace("Minus", "-") + out = out.replace("Fpix", "FPIX") + out = out.replace("Tib", "TIB") + out = out.replace("Tob", "TOB") + out = out.replace("Tid", "TID") + out = out.replace("Tec", "TEC") + out = out.replace("Layer", " L") + out = out.replace("Ring", " R") + out = out.replace("Stereo", "S") + out = out.replace("Rphi", "R") # other than Ring, this one does not add a space in front + out = out.replace("In", "i") + out = out.replace("Out", "o") + return out + + def drawTrendPlot(self, sector, coordinate, number): + self.canvas = ROOT.TCanvas("canvas%s_%s"%(sector, coordinate), "canvas", int(ROOT.gStyle.GetCanvasDefW()*3),ROOT.gStyle.GetCanvasDefH()) + ROOT.gPad.SetLeftMargin(0.06) + ROOT.gPad.SetRightMargin(0.04) + + iTrend = 0 + + if self.log: + minApe = 0.9 + maxApe = 7000.0 + ROOT.gPad.SetLogy() + else: + minApe = 0 + maxApe = 100 + + + # calibrate runrange + firstRun = 999999 + lastRun = 0 + for label, points, dashed, color, marker in self.points: + firstRun = min(min(points, key=lambda x:x[0])[0], firstRun) + lastRun = max(max(points, key=lambda x:x[1])[1], lastRun) + theFirstRun = firstRun + theLastRun = lastRun + + firstYear = whichYear(firstRun) + lastYear = whichYear(lastRun) + minLumi = 0 + + maxLumi = 0 + for year in intLumis.keys(): + if year >= firstYear and year <= lastYear: + maxLumi += intLumis[year] + + verticalLines = [] + lineLabels = [] + i = 0 + for year in range(firstYear, lastYear+1): + for position in pixelIOVs[year]: + if self.doLumi: + posLumi = runToLumi(position, firstYear, False) + else: + posLumi = position + vLine = ROOT.TLine(posLumi,minApe,posLumi,maxApe) + vLine.SetLineStyle(9) + vLine.SetLineColor(ROOT.kRed) + verticalLines.append(vLine) + + posApe = 70+3.5*(maxApe-minApe)/100*(i % 5) + + text = ROOT.TLatex(posLumi + (maxLumi-minLumi)*0.003 , posApe, str(position)) + text.SetTextFont(42) + text.SetTextSize(0.035) + text.SetTextColor(ROOT.kRed+2) + lineLabels.append(text) + i += 1 + + + legend = ROOT.TLegend(0.07, 0.89, 0.935, 0.96) + legend.SetTextFont(42) + legend.SetTextSize(0.045) + legend.SetFillStyle(0) + legend.SetBorderSize(0) + legend.SetNColumns(5) + + if self.doLumi: + hAxisLumi = ROOT.TH2F("hAxisRun%s_%s"%(sector, coordinate),"", 10, float(minLumi), float(maxLumi), 10, minApe, maxApe) + hAxisLumi.SetTitle(";integrated luminosity [fb^{-1}];#sigma_{align," + coordinate.lower() + "} [#mum]") + else: + hAxisLumi = ROOT.TH2F("hAxisRun%s_%s"%(sector, coordinate),"", 10, theFirstRun, theLastRun, 10, minApe, maxApe) + hAxisLumi.SetTitle(";Run number;#sigma_{align," + coordinate.lower() + "} [#mum]") + hAxisLumi.GetYaxis().SetTitleOffset(0.4) + hAxisLumi.GetXaxis().SetNdivisions(510) + hAxisLumi.Draw("AXIS") + trends = [] + useColor = 1 + for label, points, dashed, color, marker in self.points: + iTrend += 1 + graphLumi = ROOT.TGraphErrors() + trends.append(graphLumi) + + if color: + graphLumi.SetLineColor(color) + graphLumi.SetMarkerColor(color) + else: + while True: + if useColor not in self.colors and useColor not in [0,10]: + self.colors.add(useColor) + graphLumi.SetLineColor(useColor) + graphLumi.SetMarkerColor(useColor) + break + useColor += 1 + + if marker: + graphLumi.SetLineWidth(0) + graphLumi.SetMarkerSize(1.3) + graphLumi.SetMarkerStyle(marker) + else: + graphLumi.SetLineWidth(2) + graphLumi.SetMarkerSize(0) + graphLumi.SetMarkerStyle(20) + + + if dashed: + graphLumi.SetLineStyle(2) + + + iPoint = 0 + for firstRun, lastRun, file in points: + fi = ROOT.TFile(file, "READ") + nameTree = fi.Get("nameTree") + apeTree = fi.Get("iterTree{}".format(coordinate)) + nameTree.GetEntry(0) + apeTree.GetEntry(apeTree.GetEntries()-1) + + sectorApe = 10000. * (float(getattr(apeTree, "Ape_Sector_{}".format(sector))))**0.5 + sectorName = str(getattr(nameTree, "Ape_Sector_{}".format(sector))) + + # this could be done centrally for each trend and then not be redone for each sector + # but it does not take too much time (most time is spent reading out ROOT files) + if self.doLumi: + lumiStart = runToLumi(firstRun, firstYear, False) + lumiEnd = runToLumi(lastRun, firstYear, True) + else: + lumiStart = firstRun + lumiEnd = lastRun + + xPosLumi = (lumiStart+lumiEnd) / 2 + xErrLumi = -(lumiStart-lumiEnd) / 2 + graphLumi.SetPoint(iPoint, xPosLumi, sectorApe) + graphLumi.SetPointError(iPoint, xErrLumi,0) + + iPoint += 1 + fi.Close() + graphLumi.Draw("PZ same") + if marker: + legend.AddEntry(graphLumi, label, "pl") + else: + legend.AddEntry(graphLumi, label, "l") + cmsText = ROOT.TLatex(0.16,0.96,self.title) + cmsText.SetTextFont(42) + cmsText.SetNDC() + cmsText.Draw("same") + + sectorText = ROOT.TLatex(0.9,0.96,sectorName) + sectorText.SetTextAlign(31) + sectorText.SetTextFont(42) + sectorText.SetNDC() + sectorText.Draw("same") + + + + for vLine in verticalLines: + vLine.Draw("same") + for llabel in lineLabels: + llabel.Draw("same") + + legend.Draw("same") + + ROOT.gPad.RedrawAxis() + + import os + if not os.path.isdir("{}/{}".format(self.outPath, self.granularity.names[coordinate][number])): + os.makedirs("{}/{}".format(self.outPath, self.granularity.names[coordinate][number])) + + app = "" + if not self.doLumi: + app = "_byRun" + + self.canvas.SaveAs("{}/{}/trend_{}_{}{}.pdf".format(self.outPath, self.granularity.names[coordinate][number], coordinate, sectorName, app)) + self.canvas = None + + + + def draw(self): + for coordinate in self.granularity.sectors.keys(): + plotNumber = 0 + rangeList = self.granularity.sectors[coordinate] + for sectorRange in rangeList: + for sector in range(sectorRange[0], sectorRange[1]+1): + self.drawTrendPlot(sector, coordinate, plotNumber) + plotNumber += 1 + + +def main(): + pass + + +if __name__ == "__main__": + main() From 71e1973e8fa550524aaa7bce1d0cf2f4d33f1a2c Mon Sep 17 00:00:00 2001 From: mteroerd Date: Wed, 7 Aug 2019 22:55:20 +0200 Subject: [PATCH 2/2] Added usage of print function for python 3 compatibility --- Alignment/APEEstimation/test/plottingTools/trendPlotter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Alignment/APEEstimation/test/plottingTools/trendPlotter.py b/Alignment/APEEstimation/test/plottingTools/trendPlotter.py index d86fed8d5fc01..5ad295d106e57 100644 --- a/Alignment/APEEstimation/test/plottingTools/trendPlotter.py +++ b/Alignment/APEEstimation/test/plottingTools/trendPlotter.py @@ -1,3 +1,4 @@ +from __future__ import print_function import ROOT ROOT.gROOT.SetBatch(True) from setTDRStyle import setTDRStyle @@ -63,7 +64,7 @@ def whichYear(run): if min(runs[year]) <= run: thisYear = year return thisYear - print "Run %d not in range of any year"%(run) + print("Run %d not in range of any year"%(run)) return -1